PROTOBUF_SYNC_PIPER
pull/10360/head
Matt Kulukundis 3 years ago
parent c7b2991888
commit e8ab276325
  1. 144
      cmake/extract_includes.bat.in
  2. 6
      csharp/src/Google.Protobuf/JsonFormatter.cs
  3. 1
      docs/third_party.md
  4. 21
      kokoro/linux/cmake_shared/build.sh
  5. 41
      php/ext/google/protobuf/def.c
  6. 20
      php/tests/DescriptorsTest.php
  7. 1
      php/tests/GeneratedClassTest.php
  8. 1
      php/tests/proto/test_descriptors.proto
  9. 2
      php/tests/proto/test_php_namespace.proto
  10. 8
      python/google/protobuf/descriptor.py
  11. 13
      python/google/protobuf/internal/message_test.py
  12. 46
      src/google/protobuf/arenaz_sampler.cc
  13. 35
      src/google/protobuf/arenaz_sampler.h
  14. 147
      src/google/protobuf/arenaz_sampler_test.cc
  15. 3
      src/google/protobuf/compiler/command_line_interface.cc
  16. 2
      src/google/protobuf/compiler/cpp/options.h
  17. 5
      src/google/protobuf/compiler/java/file.cc
  18. 33
      src/google/protobuf/compiler/php/php_generator.cc
  19. 146
      src/google/protobuf/compiler/python/generator.cc
  20. 1
      src/google/protobuf/compiler/python/generator.h
  21. 6
      src/google/protobuf/descriptor.h
  22. 5
      src/google/protobuf/message.h
  23. 8
      src/google/protobuf/message_unittest.inc
  24. 53
      src/google/protobuf/repeated_field.h
  25. 54
      src/google/protobuf/repeated_field_unittest.cc

@ -1,144 +0,0 @@
mkdir include
mkdir include\google
mkdir include\google\protobuf
mkdir include\google\protobuf\compiler
mkdir include\google\protobuf\compiler\cpp
mkdir include\google\protobuf\compiler\csharp
mkdir include\google\protobuf\compiler\java
mkdir include\google\protobuf\compiler\objectivec
mkdir include\google\protobuf\compiler\php
mkdir include\google\protobuf\compiler\python
mkdir include\google\protobuf\compiler\ruby
mkdir include\google\protobuf\io
mkdir include\google\protobuf\stubs
mkdir include\google\protobuf\util
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.h" include\google\protobuf\any.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.pb.h" include\google\protobuf\any.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.pb.h" include\google\protobuf\api.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena.h" include\google\protobuf\arena.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena_impl.h" include\google\protobuf\arena_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenastring.h" include\google\protobuf\arenastring.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenaz_sampler.h" include\google\protobuf\arenaz_sampler.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\code_generator.h" include\google\protobuf\compiler\code_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\command_line_interface.h" include\google\protobuf\compiler\command_line_interface.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\file.h" include\google\protobuf\compiler\cpp\file.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\cpp_generator.h" include\google\protobuf\compiler\cpp\cpp_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\generator.h" include\google\protobuf\compiler\cpp\generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\helpers.h" include\google\protobuf\compiler\cpp\helpers.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\names.h" include\google\protobuf\compiler\cpp\names.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_doc_comment.h" include\google\protobuf\compiler\csharp\csharp_doc_comment.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_generator.h" include\google\protobuf\compiler\csharp\csharp_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_names.h" include\google\protobuf\compiler\csharp\csharp_names.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_options.h" include\google\protobuf\compiler\csharp\csharp_options.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\importer.h" include\google\protobuf\compiler\importer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\generator.h" include\google\protobuf\compiler\java\generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_generator.h" include\google\protobuf\compiler\java\java_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\kotlin_generator.h" include\google\protobuf\compiler\java\kotlin_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\names.h" include\google\protobuf\compiler\java\names.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_generator.h" include\google\protobuf\compiler\objectivec\objectivec_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_helpers.h" include\google\protobuf\compiler\objectivec\objectivec_helpers.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\parser.h" include\google\protobuf\compiler\parser.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\php\php_generator.h" include\google\protobuf\compiler\php\php_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.h" include\google\protobuf\compiler\plugin.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.pb.h" include\google\protobuf\compiler\plugin.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\generator.h" include\google\protobuf\compiler\python\generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\pyi_generator.h" include\google\protobuf\compiler\python\pyi_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\python_generator.h" include\google\protobuf\compiler\python\python_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\ruby\ruby_generator.h" include\google\protobuf\compiler\ruby\ruby_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.h" include\google\protobuf\descriptor.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.pb.h" include\google\protobuf\descriptor.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor_database.h" include\google\protobuf\descriptor_database.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.pb.h" include\google\protobuf\duration.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\dynamic_message.h" include\google\protobuf\dynamic_message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.pb.h" include\google\protobuf\empty.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\endian.h" include\google\protobuf\endian.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\explicitly_constructed.h" include\google\protobuf\explicitly_constructed.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set.h" include\google\protobuf\extension_set.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set_inl.h" include\google\protobuf\extension_set_inl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_access_listener.h" include\google\protobuf\field_access_listener.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.pb.h" include\google\protobuf\field_mask.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_reflection.h" include\google\protobuf\generated_enum_reflection.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_util.h" include\google\protobuf\generated_enum_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_bases.h" include\google\protobuf\generated_message_bases.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_reflection.h" include\google\protobuf\generated_message_reflection.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_decl.h" include\google\protobuf\generated_message_tctable_decl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_impl.h" include\google\protobuf\generated_message_tctable_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\inlined_string_field.h" include\google\protobuf\inlined_string_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\printer.h" include\google\protobuf\io\printer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\strtod.h" include\google\protobuf\io\strtod.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\tokenizer.h" include\google\protobuf\io\tokenizer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream.h" include\google\protobuf\io\zero_copy_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream_impl.h" include\google\protobuf\io\zero_copy_stream_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream_impl_lite.h" include\google\protobuf\io\zero_copy_stream_impl_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map.h" include\google\protobuf\map.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_entry.h" include\google\protobuf\map_entry.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_entry_lite.h" include\google\protobuf\map_entry_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field.h" include\google\protobuf\map_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field_inl.h" include\google\protobuf\map_field_inl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field_lite.h" include\google\protobuf\map_field_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_type_handler.h" include\google\protobuf\map_type_handler.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\message.h" include\google\protobuf\message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\message_lite.h" include\google\protobuf\message_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\metadata.h" include\google\protobuf\metadata.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\metadata_lite.h" include\google\protobuf\metadata_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\parse_context.h" include\google\protobuf\parse_context.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port.h" include\google\protobuf\port.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_def.inc" include\google\protobuf\port_def.inc
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_undef.inc" include\google\protobuf\port_undef.inc
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection.h" include\google\protobuf\reflection.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection_ops.h" include\google\protobuf\reflection_ops.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" include\google\protobuf\repeated_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_ptr_field.h" include\google\protobuf\repeated_ptr_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" include\google\protobuf\stubs\bytestream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\map_util.h" include\google\protobuf\stubs\map_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\mutex.h" include\google\protobuf\stubs\mutex.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\once.h" include\google\protobuf\stubs\once.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\platform_macros.h" include\google\protobuf\stubs\platform_macros.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\port.h" include\google\protobuf\stubs\port.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\status.h" include\google\protobuf\stubs\status.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stl_util.h" include\google\protobuf\stubs\stl_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stringpiece.h" include\google\protobuf\stubs\stringpiece.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\strutil.h" include\google\protobuf\stubs\strutil.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\template_util.h" include\google\protobuf\stubs\template_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\text_format.h" include\google\protobuf\text_format.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\timestamp.pb.h" include\google\protobuf\timestamp.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\type.pb.h" include\google\protobuf\type.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\unknown_field_set.h" include\google\protobuf\unknown_field_set.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\delimited_message_util.h" include\google\protobuf\util\delimited_message_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\field_comparator.h" include\google\protobuf\util\field_comparator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\field_mask_util.h" include\google\protobuf\util\field_mask_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\json_util.h" include\google\protobuf\util\json_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\message_differencer.h" include\google\protobuf\util\message_differencer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\time_util.h" include\google\protobuf\util\time_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver.h" include\google\protobuf\util\type_resolver.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver_util.h" include\google\protobuf\util\type_resolver_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format.h" include\google\protobuf\wire_format.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite.h" include\google\protobuf\wire_format_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.pb.h" include\google\protobuf\wrappers.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.proto" include\google\protobuf\any.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.proto" include\google\protobuf\api.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.proto" include\google\protobuf\compiler\plugin.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.proto" include\google\protobuf\descriptor.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.proto" include\google\protobuf\duration.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.proto" include\google\protobuf\empty.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.proto" include\google\protobuf\field_mask.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.proto" include\google\protobuf\source_context.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.proto" include\google\protobuf\struct.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\timestamp.proto" include\google\protobuf\timestamp.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\type.proto" include\google\protobuf\type.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.proto" include\google\protobuf\wrappers.proto

@ -154,7 +154,7 @@ namespace Google.Protobuf
/// </summary>
/// <param name="message">The message to format.</param>
/// <param name="indentationLevel">Indentation level to start at.</param>
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <see cref="indentationLevel"/>. E.g:
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <paramref name="indentationLevel"/>. E.g:
/// <code>
/// var response = $@"{{
/// ""data"": { Format(message, indentationLevel: 1) }
@ -178,12 +178,12 @@ namespace Google.Protobuf
public void Format(IMessage message, TextWriter writer) => Format(message, writer, indentationLevel: 0);
/// <summary>
/// Formats the specified message as JSON. When <see cref="Settings.Indentation"/> is not null, start indenting at the specified <see cref="indentationLevel"/>.
/// Formats the specified message as JSON. When <see cref="Settings.Indentation"/> is not null, start indenting at the specified <paramref name="indentationLevel"/>.
/// </summary>
/// <param name="message">The message to format.</param>
/// <param name="writer">The TextWriter to write the formatted message to.</param>
/// <param name="indentationLevel">Indentation level to start at.</param>
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <see cref="indentationLevel"/>.</remarks>
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <paramref name="indentationLevel"/>.</remarks>
public void Format(IMessage message, TextWriter writer, int indentationLevel)
{
ProtoPreconditions.CheckNotNull(message, nameof(message));

@ -203,3 +203,4 @@ There are miscellaneous other things you may find useful as a Protocol Buffers d
* [vim-protolint: A protobuf linter for Vim](https://github.com/yoheimuta/vim-protolint)
* [super-linter: Protocol Buffer lint as GitHub Action](https://github.com/github/super-linter)
* [protoc-gen-fieldmask - A plugin to generate static type fieldmask paths](https://github.com/idodod/protoc-gen-fieldmask)
* [protoc-gen-bq-schema - A protoc plugin to generate BigQuery schema files](https://github.com/GoogleCloudPlatform/protoc-gen-bq-schema)

@ -4,4 +4,23 @@
set -eux
# TODO(mkruskal) Implement this.
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=`pwd`
CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
# Update git submodules
git submodule update --init --recursive
tmpfile=$(mktemp -u)
docker run \
--cidfile $tmpfile \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
/test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_SHARED_LIBS=ON
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

@ -456,6 +456,44 @@ PHP_METHOD(FieldDescriptor, getEnumType) {
RETURN_COPY_VALUE(&ret);
}
/*
* FieldDescriptor::getContainingOneof()
*
* Returns the OneofDescriptor for this field, or null if it is not inside
* a oneof.
*/
PHP_METHOD(FieldDescriptor, getContainingOneof) {
FieldDescriptor *intern = (FieldDescriptor*)Z_OBJ_P(getThis());
const upb_OneofDef *o = upb_FieldDef_ContainingOneof(intern->fielddef);
zval ret;
if (!o) {
RETURN_NULL();
}
OneofDescriptor_FromOneofDef(&ret, o);
RETURN_COPY_VALUE(&ret);
}
/*
* FieldDescriptor::getRealContainingOneof()
*
* Returns the non-synthetic OneofDescriptor for this field, or null if it is
* not inside a oneof.
*/
PHP_METHOD(FieldDescriptor, getRealContainingOneof) {
FieldDescriptor *intern = (FieldDescriptor*)Z_OBJ_P(getThis());
const upb_OneofDef *o = upb_FieldDef_RealContainingOneof(intern->fielddef);
zval ret;
if (!o) {
RETURN_NULL();
}
OneofDescriptor_FromOneofDef(&ret, o);
RETURN_COPY_VALUE(&ret);
}
/*
* FieldDescriptor::getMessageType()
*
@ -482,6 +520,8 @@ static zend_function_entry FieldDescriptor_methods[] = {
PHP_ME(FieldDescriptor, getType, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, isMap, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getEnumType, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getContainingOneof, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getRealContainingOneof, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getMessageType, arginfo_void, ZEND_ACC_PUBLIC)
ZEND_FE_END
};
@ -516,6 +556,7 @@ static zend_class_entry *Descriptor_GetGeneratedClass(const upb_MessageDef *m) {
char *classname =
GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m), false);
zend_error(E_ERROR, "Couldn't load generated class %s", classname);
return NULL;
}
void Descriptor_FromMessageDef(zval *val, const upb_MessageDef *m) {

@ -83,10 +83,10 @@ class DescriptorsTest extends TestBase
$this->assertSame($class, $desc->getClass());
$this->assertInstanceOf('\Google\Protobuf\FieldDescriptor', $desc->getField(0));
$this->assertSame(7, $desc->getFieldCount());
$this->assertSame(8, $desc->getFieldCount());
$this->assertInstanceOf('\Google\Protobuf\OneofDescriptor', $desc->getOneofDecl(0));
$this->assertSame(1, $desc->getOneofDeclCount());
$this->assertSame(2, $desc->getOneofDeclCount());
}
public function testDescriptorForIncludedMessage()
@ -180,6 +180,7 @@ class DescriptorsTest extends TestBase
$this->assertSame(self::GPBTYPE_MESSAGE, $fieldDesc->getType());
$this->assertInstanceOf('\Google\Protobuf\Descriptor', $fieldDesc->getMessageType());
$this->assertFalse($fieldDesc->isMap());
$this->assertNull($fieldDesc->getContainingOneof());
// Oneof int field
// Tested further in testOneofDescriptor()
@ -189,6 +190,21 @@ class DescriptorsTest extends TestBase
$this->assertSame(self::GPBLABEL_OPTIONAL, $fieldDesc->getLabel());
$this->assertSame(self::GPBTYPE_INT32, $fieldDesc->getType());
$this->assertFalse($fieldDesc->isMap());
$this->assertSame($fieldDesc->getContainingOneof(), $fieldDesc->getRealContainingOneof());
$oneofDesc = $fieldDesc->getContainingOneof();
$this->assertSame('my_oneof', $oneofDesc->getName());
// Proto3 optional it field.
// Tested further in testOneofDescriptor()
$fieldDesc = $fieldDescMap[52];
$this->assertSame('proto3_optional_int32', $fieldDesc->getName());
$this->assertSame(52, $fieldDesc->getNumber());
$this->assertSame(self::GPBLABEL_OPTIONAL, $fieldDesc->getLabel());
$this->assertSame(self::GPBTYPE_INT32, $fieldDesc->getType());
$this->assertFalse($fieldDesc->isMap());
$this->assertNull($fieldDesc->getRealContainingOneof());
$this->assertNotNull($fieldDesc->getContainingOneof());
// Map int-enum field
$fieldDesc = $fieldDescMap[71];

@ -338,6 +338,7 @@ class GeneratedClassTest extends TestBase
{
$this->assertTrue(class_exists('\Upper\READONLY'));
$this->assertTrue(class_exists('\Lower\readonly'));
$this->assertTrue(class_exists('\Php\Test\TestNamespace\PBEmpty\ReadOnly'));
}
public function testLegacyReadOnlyEnum()

@ -14,6 +14,7 @@ message TestDescriptorsMessage {
oneof my_oneof {
int32 oneof_int32 = 51;
}
optional int32 proto3_optional_int32 = 52;
map<int32, EnumSub> map_int32_enum = 71;

@ -27,5 +27,7 @@ message TestNamespace {
enum NestedEnum {
ZERO = 0;
};
// Test previously unreserved name
message ReadOnly {}
}
}

@ -1021,13 +1021,7 @@ class FileDescriptor(DescriptorBase):
# FileDescriptor() is called from various places, not only from generated
# files, to register dynamic proto files and messages.
# pylint: disable=g-explicit-bool-comparison
if serialized_pb == b'':
# Cpp generated code must be linked in if serialized_pb is ''
try:
return _message.default_pool.FindFileByName(name)
except KeyError:
raise RuntimeError('Please link in cpp generated lib for %s' % (name))
elif serialized_pb:
if serialized_pb:
return _message.default_pool.AddSerializedFile(serialized_pb)
else:
return super(FileDescriptor, cls).__new__(cls)

@ -1768,6 +1768,19 @@ class Proto3Test(unittest.TestCase):
with self.assertRaises(TypeError):
123 in msg.map_string_string
def testScalarMapComparison(self):
msg1 = map_unittest_pb2.TestMap()
msg2 = map_unittest_pb2.TestMap()
self.assertEqual(msg1.map_int32_int32, msg2.map_int32_int32)
def testMessageMapComparison(self):
msg1 = map_unittest_pb2.TestMap()
msg2 = map_unittest_pb2.TestMap()
self.assertEqual(msg1.map_int32_foreign_message,
msg2.map_int32_foreign_message)
def testMapGet(self):
# Need to test that get() properly returns the default, even though the dict
# has defaultdict-like semantics.

@ -33,6 +33,7 @@
#include <atomic>
#include <cstdint>
#include <limits>
#include <utility>
// Must be included last.
@ -74,11 +75,15 @@ PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = {
ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); }
ThreadSafeArenaStats::~ThreadSafeArenaStats() = default;
void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
void ThreadSafeArenaStats::BlockStats::PrepareForSampling() {
num_allocations.store(0, std::memory_order_relaxed);
bytes_used.store(0, std::memory_order_relaxed);
bytes_allocated.store(0, std::memory_order_relaxed);
bytes_used.store(0, std::memory_order_relaxed);
bytes_wasted.store(0, std::memory_order_relaxed);
}
void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
for (auto& blockstats : block_histogram) blockstats.PrepareForSampling();
max_block_size.store(0, std::memory_order_relaxed);
thread_ids.store(0, std::memory_order_relaxed);
weight = stride;
@ -88,12 +93,41 @@ void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
depth = absl::GetStackTrace(stack, kMaxStackDepth, /* skip_count= */ 0);
}
size_t ThreadSafeArenaStats::FindBin(size_t bytes) {
if (bytes <= kMaxSizeForBinZero) return 0;
if (bytes <= kMaxSizeForPenultimateBin) {
// absl::bit_width() returns one plus the base-2 logarithm of x, with any
// fractional part discarded.
return absl::bit_width(absl::bit_ceil(bytes)) - kLogMaxSizeForBinZero - 1;
}
return kBlockHistogramBins - 1;
}
std::pair<size_t, size_t> ThreadSafeArenaStats::MinMaxBlockSizeForBin(
size_t bin) {
ABSL_ASSERT(bin < kBlockHistogramBins);
if (bin == 0) return {1, kMaxSizeForBinZero};
if (bin < kBlockHistogramBins - 1) {
return {(1 << (kLogMaxSizeForBinZero + bin - 1)) + 1,
1 << (kLogMaxSizeForBinZero + bin)};
}
return {kMaxSizeForPenultimateBin + 1, std::numeric_limits<size_t>::max()};
}
void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t used,
size_t allocated, size_t wasted) {
info->num_allocations.fetch_add(1, std::memory_order_relaxed);
info->bytes_used.fetch_add(used, std::memory_order_relaxed);
info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
// Update the allocated bytes for the current block.
ThreadSafeArenaStats::BlockStats& curr =
info->block_histogram[ThreadSafeArenaStats::FindBin(allocated)];
curr.bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
curr.num_allocations.fetch_add(1, std::memory_order_relaxed);
// Update the used and wasted bytes for the previous block.
ThreadSafeArenaStats::BlockStats& prev =
info->block_histogram[ThreadSafeArenaStats::FindBin(used + wasted)];
prev.bytes_used.fetch_add(used, std::memory_order_relaxed);
prev.bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
if (info->max_block_size.load(std::memory_order_relaxed) < allocated) {
info->max_block_size.store(allocated, std::memory_order_relaxed);
}

@ -31,9 +31,11 @@
#ifndef GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__
#define GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__
#include <array>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <utility>
// Must be included last.
@ -65,10 +67,28 @@ struct ThreadSafeArenaStats
// These fields are mutated by the various Record* APIs and need to be
// thread-safe.
std::atomic<int> num_allocations;
std::atomic<size_t> bytes_used;
std::atomic<size_t> bytes_allocated;
std::atomic<size_t> bytes_wasted;
struct BlockStats {
std::atomic<int> num_allocations;
std::atomic<size_t> bytes_allocated;
std::atomic<size_t> bytes_used;
std::atomic<size_t> bytes_wasted;
void PrepareForSampling();
};
// block_histogram is a kBlockHistogramBins sized histogram. The zeroth bin
// stores info about blocks of size \in [1, 1 << kLogMaxSizeForBinZero]. Bin
// i, where i > 0, stores info for blocks of size \in (max_size_bin (i-1),
// 1 << (kLogMaxSizeForBinZero + i)]. The final bin stores info about blocks
// of size \in [kMaxSizeForPenultimateBin + 1,
// std::numeric_limits<size_t>::max()].
static constexpr size_t kBlockHistogramBins = 15;
static constexpr size_t kLogMaxSizeForBinZero = 7;
static constexpr size_t kMaxSizeForBinZero = (1 << kLogMaxSizeForBinZero);
static constexpr size_t kMaxSizeForPenultimateBin =
1 << (kLogMaxSizeForBinZero + kBlockHistogramBins - 2);
std::array<BlockStats, kBlockHistogramBins> block_histogram;
// Records the largest block allocated for the arena.
std::atomic<size_t> max_block_size;
// Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed
@ -90,6 +110,13 @@ struct ThreadSafeArenaStats
if (PROTOBUF_PREDICT_TRUE(info == nullptr)) return;
RecordAllocateSlow(info, used, allocated, wasted);
}
// Returns the bin for the provided size.
static size_t FindBin(size_t bytes);
// Returns the min and max bytes that can be stored in the histogram for
// blocks in the provided bin.
static std::pair<size_t, size_t> MinMaxBlockSizeForBin(size_t bin);
};
struct SamplingState {

@ -30,6 +30,8 @@
#include <google/protobuf/arenaz_sampler.h>
#include <atomic>
#include <limits>
#include <memory>
#include <random>
#include <utility>
@ -37,7 +39,6 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/strutil.h>
// Must be included last.
@ -57,10 +58,17 @@ class ThreadSafeArenaStatsHandlePeer {
return h->info_;
}
};
std::vector<size_t> GetBytesAllocated(ThreadSafeArenazSampler* s) {
std::vector<size_t> res;
s->Iterate([&](const ThreadSafeArenaStats& info) {
res.push_back(info.bytes_allocated.load(std::memory_order_acquire));
for (const auto& block_stats : info.block_histogram) {
size_t bytes_allocated =
block_stats.bytes_allocated.load(std::memory_order_acquire);
if (bytes_allocated != 0) {
res.push_back(bytes_allocated);
}
}
});
return res;
}
@ -69,7 +77,8 @@ ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size,
int64_t stride) {
auto* info = s->Register(stride);
assert(info != nullptr);
info->bytes_allocated.store(size);
info->block_histogram[0].bytes_allocated.store(size,
std::memory_order_relaxed);
return info;
}
@ -85,46 +94,100 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) {
MutexLock l(&info.init_mu);
info.PrepareForSampling(kTestStride);
EXPECT_EQ(info.num_allocations.load(), 0);
EXPECT_EQ(info.bytes_used.load(), 0);
EXPECT_EQ(info.bytes_allocated.load(), 0);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_block_size.load(), 0);
for (const auto& block_stats : info.block_histogram) {
EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0);
}
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0);
EXPECT_EQ(info.weight, kTestStride);
info.num_allocations.store(1, std::memory_order_relaxed);
info.bytes_used.store(1, std::memory_order_relaxed);
info.bytes_allocated.store(1, std::memory_order_relaxed);
info.bytes_wasted.store(1, std::memory_order_relaxed);
for (auto& block_stats : info.block_histogram) {
block_stats.num_allocations.store(1, std::memory_order_relaxed);
block_stats.bytes_used.store(1, std::memory_order_relaxed);
block_stats.bytes_allocated.store(1, std::memory_order_relaxed);
block_stats.bytes_wasted.store(1, std::memory_order_relaxed);
}
info.max_block_size.store(1, std::memory_order_relaxed);
info.PrepareForSampling(2 * kTestStride);
EXPECT_EQ(info.num_allocations.load(), 0);
EXPECT_EQ(info.bytes_used.load(), 0);
EXPECT_EQ(info.bytes_allocated.load(), 0);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_block_size.load(), 0);
for (auto& block_stats : info.block_histogram) {
EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0);
}
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0);
EXPECT_EQ(info.weight, 2 * kTestStride);
}
TEST(ThreadSafeArenaStatsTest, FindBin) {
size_t current_bin = 0;
size_t bytes = 1;
while (current_bin < ThreadSafeArenaStats::kBlockHistogramBins - 1) {
size_t next_bin = ThreadSafeArenaStats::FindBin(bytes);
if (next_bin != current_bin) {
// Test the bins increase linearly.
EXPECT_EQ(next_bin, current_bin + 1);
// Test the bins change only at values of the form 2^k + 1.
EXPECT_EQ(absl::popcount(bytes - 1), 1);
current_bin = next_bin;
}
++bytes;
}
}
TEST(ThreadSafeArenaStatsTest, MinMaxBlockSizeForBin) {
std::pair<size_t, size_t> current_limits =
ThreadSafeArenaStats::MinMaxBlockSizeForBin(0);
EXPECT_EQ(current_limits.first, 1);
EXPECT_LT(current_limits.first, current_limits.second);
for (size_t i = 1; i < ThreadSafeArenaStats::kBlockHistogramBins; ++i) {
std::pair<size_t, size_t> next_limits =
ThreadSafeArenaStats::MinMaxBlockSizeForBin(i);
EXPECT_LT(next_limits.first, next_limits.second);
// Test the limits do not have gaps.
EXPECT_EQ(next_limits.first, current_limits.second + 1);
if (i != ThreadSafeArenaStats::kBlockHistogramBins - 1) {
EXPECT_EQ(next_limits.second, 2 * current_limits.second);
}
current_limits = next_limits;
}
// Test the limits cover the entire range possible.
EXPECT_EQ(current_limits.second, std::numeric_limits<size_t>::max());
}
TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) {
ThreadSafeArenaStats info;
constexpr int64_t kTestStride = 458;
MutexLock l(&info.init_mu);
info.PrepareForSampling(kTestStride);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(info.num_allocations.load(), 1);
EXPECT_EQ(info.bytes_used.load(), 100);
EXPECT_EQ(info.bytes_allocated.load(), 128);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_block_size.load(), 128);
RecordAllocateSlow(&info, /*requested=*/0, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(
info.block_histogram[0].num_allocations.load(std::memory_order_relaxed),
1);
EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed),
0);
EXPECT_EQ(
info.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed),
128);
EXPECT_EQ(
info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 0);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256,
/*wasted=*/28);
EXPECT_EQ(info.num_allocations.load(), 2);
EXPECT_EQ(info.bytes_used.load(), 200);
EXPECT_EQ(info.bytes_allocated.load(), 384);
EXPECT_EQ(info.bytes_wasted.load(), 28);
EXPECT_EQ(info.max_block_size.load(), 256);
EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed),
100);
EXPECT_EQ(
info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 28);
EXPECT_EQ(
info.block_histogram[1].num_allocations.load(std::memory_order_relaxed),
1);
EXPECT_EQ(
info.block_histogram[1].bytes_allocated.load(std::memory_order_relaxed),
256);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256);
}
TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) {
@ -133,13 +196,13 @@ TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) {
MutexLock l(&info.init_mu);
info.PrepareForSampling(kTestStride);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(info.max_block_size.load(), 128);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256,
/*wasted=*/28);
EXPECT_EQ(info.max_block_size.load(), 256);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128,
/*wasted=*/28);
EXPECT_EQ(info.max_block_size.load(), 256);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256);
}
TEST(ThreadSafeArenazSamplerTest, SamplingCorrectness) {
@ -212,12 +275,15 @@ TEST(ThreadSafeArenazSamplerTest, Handle) {
constexpr int64_t kTestStride = 17;
ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride));
auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h);
info->bytes_allocated.store(0x12345678, std::memory_order_relaxed);
info->block_histogram[0].bytes_allocated.store(0x12345678,
std::memory_order_relaxed);
bool found = false;
sampler.Iterate([&](const ThreadSafeArenaStats& h) {
if (&h == info) {
EXPECT_EQ(h.bytes_allocated.load(), 0x12345678);
EXPECT_EQ(
h.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed),
0x12345678);
EXPECT_EQ(h.weight, kTestStride);
found = true;
}
@ -230,7 +296,8 @@ TEST(ThreadSafeArenazSamplerTest, Handle) {
if (&h == info) {
// this will only happen if some other thread has resurrected the info
// the old handle was using.
if (h.bytes_allocated.load() == 0x12345678) {
if (h.block_histogram[0].bytes_allocated.load(
std::memory_order_relaxed) == 0x12345678) {
found = true;
}
}
@ -246,7 +313,7 @@ TEST(ThreadSafeArenazSamplerTest, Registration) {
auto* info2 = Register(&sampler, 2, kTestStride);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2));
info1->bytes_allocated.store(3);
info1->block_histogram[0].bytes_allocated.store(3, std::memory_order_relaxed);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2));
sampler.Unregister(info1);
@ -258,18 +325,18 @@ TEST(ThreadSafeArenazSamplerTest, Unregistration) {
std::vector<ThreadSafeArenaStats*> infos;
constexpr int64_t kTestStride = 200;
for (size_t i = 0; i < 3; ++i) {
infos.push_back(Register(&sampler, i, kTestStride));
infos.push_back(Register(&sampler, i + 1, kTestStride));
}
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2, 3));
sampler.Unregister(infos[1]);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3));
infos.push_back(Register(&sampler, 3, kTestStride));
infos.push_back(Register(&sampler, 4, kTestStride));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 3, 4));
sampler.Unregister(infos[3]);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 4));
sampler.Unregister(infos[0]);
sampler.Unregister(infos[2]);

@ -2124,7 +2124,8 @@ bool CommandLineInterface::EnforceProto3OptionalSupport(
<< codegen_name
<< " hasn't been updated to support optional fields in "
"proto3. Please ask the owner of this code generator to "
"support proto3 optional.";
"support proto3 optional."
<< std::endl;
return false;
}
}

@ -85,7 +85,7 @@ struct Options {
bool profile_driven_inline_string = true;
bool message_owned_arena_trial = false;
bool force_split = false;
bool profile_driven_split = false;
bool profile_driven_split = true;
#ifdef PROTOBUF_STABLE_EXPERIMENTS
bool force_eagerly_verified_lazy = true;
bool force_inline_string = true;

@ -704,10 +704,13 @@ void FileGenerator::GenerateKotlinSiblings(
options_.annotate_code ? &annotation_collector : nullptr);
printer.Print(
"//Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", descriptor->file()->name());
printer.Print(
"// Generated files should ignore deprecation warnings\n"
"@file:Suppress(\"DEPRECATION\")");
if (!java_package_.empty()) {
printer.Print(
"package $package$;\n"

@ -421,12 +421,17 @@ std::string LegacyGeneratedClassFileName(const DescriptorType* desc,
}
template <typename DescriptorType>
std::string LegacyReadOnlyGeneratedClassFileName(const DescriptorType* desc,
const Options& options) {
std::string php_namespace = RootPhpNamespace(desc, options);
std::string LegacyReadOnlyGeneratedClassFileName(std::string php_namespace,
const DescriptorType* desc) {
if (!php_namespace.empty()) {
for (int i = 0; i < php_namespace.size(); i++) {
if (php_namespace[i] == '\\') {
php_namespace[i] = '/';
}
}
return php_namespace + "/" + desc->name() + ".php";
}
return desc->name() + ".php";
}
@ -1316,27 +1321,39 @@ template <typename DescriptorType>
void LegacyReadOnlyGenerateClassFile(const FileDescriptor* file,
const DescriptorType* desc, const Options& options,
GeneratorContext* generator_context) {
std::string filename = LegacyReadOnlyGeneratedClassFileName(desc, options);
std::string fullname = FullClassName(desc, options);
std::string php_namespace;
std::string classname;
int lastindex = fullname.find_last_of("\\");
if (lastindex != std::string::npos) {
php_namespace = fullname.substr(0, lastindex);
classname = fullname.substr(lastindex + 1);
} else {
php_namespace = "";
classname = fullname;
}
std::string filename = LegacyReadOnlyGeneratedClassFileName(php_namespace, desc);
std::unique_ptr<io::ZeroCopyOutputStream> output(
generator_context->Open(filename));
io::Printer printer(output.get(), '^');
GenerateHead(file, &printer);
std::string php_namespace = RootPhpNamespace(desc, options);
if (!php_namespace.empty()) {
printer.Print(
"namespace ^name^;\n\n",
"name", php_namespace);
}
std::string newname = FullClassName(desc, options);
printer.Print("class_exists(^new^::class); // autoload the new class, which "
"will also create an alias to the deprecated class\n",
"new", GeneratedClassNameImpl(desc));
"new", classname);
printer.Print("@trigger_error(__NAMESPACE__ . '\\^old^ is deprecated and will be removed in "
"the next major release. Use ^fullname^ instead', E_USER_DEPRECATED);\n\n",
"old", desc->name(),
"fullname", newname);
"fullname", classname);
}
void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en,

@ -217,15 +217,12 @@ bool Generator::Generate(const FileDescriptor* file,
GeneratorContext* context, std::string* error) const {
// -----------------------------------------------------------------
// parse generator options
bool cpp_generated_lib_linked = false;
std::vector<std::pair<std::string, std::string> > options;
ParseGeneratorParameter(parameter, &options);
for (int i = 0; i < options.size(); i++) {
if (options[i].first == "cpp_generated_lib_linked") {
cpp_generated_lib_linked = true;
} else if (options[i].first == "pyi_out") {
if (options[i].first == "pyi_out") {
python::PyiGenerator pyi_generator;
if (!pyi_generator.Generate(file, "", context, error)) {
return false;
@ -247,10 +244,6 @@ bool Generator::Generate(const FileDescriptor* file,
file_ = file;
std::string filename = GetFileName(file, ".py");
pure_python_workable_ = !cpp_generated_lib_linked;
if (HasPrefixString(file->name(), "google/protobuf/")) {
pure_python_workable_ = true;
}
FileDescriptorProto fdp;
file_->CopyTo(&fdp);
@ -263,55 +256,49 @@ bool Generator::Generate(const FileDescriptor* file,
printer_ = &printer;
PrintTopBoilerplate(printer_, file_, GeneratingDescriptorProto());
if (pure_python_workable_) {
PrintImports();
}
PrintImports();
PrintFileDescriptor();
if (pure_python_workable_) {
if (GeneratingDescriptorProto()) {
printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// Create enums before message descriptors
PrintAllNestedEnumsInFile();
PrintMessageDescriptors();
FixForeignFieldsInDescriptors();
printer_->Outdent();
printer_->Print("else:\n");
printer_->Indent();
}
// Find the message descriptors first and then use the message
// descriptor to find enums.
printer_->Print(
"_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n");
if (GeneratingDescriptorProto()) {
printer_->Outdent();
}
if (GeneratingDescriptorProto()) {
printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// Create enums before message descriptors
PrintAllNestedEnumsInFile();
PrintMessageDescriptors();
FixForeignFieldsInDescriptors();
printer_->Outdent();
printer_->Print("else:\n");
printer_->Indent();
}
// Find the message descriptors first and then use the message
// descriptor to find enums.
printer_->Print(
"_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n");
if (GeneratingDescriptorProto()) {
printer_->Outdent();
}
std::string module_name = ModuleName(file->name());
printer_->Print(
"_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '$module_name$', "
"globals())\n",
"module_name", module_name);
if (pure_python_workable_) {
printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// We have to fix up the extensions after the message classes themselves,
// since they need to call static RegisterExtension() methods on these
// classes.
FixForeignFieldsInExtensions();
// Descriptor options may have custom extensions. These custom options
// can only be successfully parsed after we register corresponding
// extensions. Therefore we parse all options again here to recognize
// custom options that may be unknown when we define the descriptors.
// This does not apply to services because they are not used by extensions.
FixAllDescriptorOptions();
// We have to fix up the extensions after the message classes themselves,
// since they need to call static RegisterExtension() methods on these
// classes.
FixForeignFieldsInExtensions();
// Descriptor options may have custom extensions. These custom options
// can only be successfully parsed after we register corresponding
// extensions. Therefore we parse all options again here to recognize
// custom options that may be unknown when we define the descriptors.
// This does not apply to services because they are not used by extensions.
FixAllDescriptorOptions();
// Set serialized_start and serialized_end.
SetSerializedPbInterval();
// Set serialized_start and serialized_end.
SetSerializedPbInterval();
printer_->Outdent();
}
printer_->Outdent();
if (HasGenericServices(file)) {
printer_->Print(
"_builder.BuildServices(DESCRIPTOR, '$module_name$', globals())\n",
@ -389,30 +376,26 @@ void Generator::PrintFileDescriptor() const {
" create_key=_descriptor._internal_create_key,\n";
printer_->Print(m, file_descriptor_template);
printer_->Indent();
if (pure_python_workable_) {
printer_->Print("serialized_pb=b'$value$'\n", "value",
strings::CHexEscape(file_descriptor_serialized_));
if (file_->dependency_count() != 0) {
printer_->Print(",\ndependencies=[");
for (int i = 0; i < file_->dependency_count(); ++i) {
std::string module_alias = ModuleAlias(file_->dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
printer_->Print("]");
printer_->Print("serialized_pb=b'$value$'\n", "value",
strings::CHexEscape(file_descriptor_serialized_));
if (file_->dependency_count() != 0) {
printer_->Print(",\ndependencies=[");
for (int i = 0; i < file_->dependency_count(); ++i) {
std::string module_alias = ModuleAlias(file_->dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
if (file_->public_dependency_count() > 0) {
printer_->Print(",\npublic_dependencies=[");
for (int i = 0; i < file_->public_dependency_count(); ++i) {
std::string module_alias =
ModuleAlias(file_->public_dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
printer_->Print("]");
printer_->Print("]");
}
if (file_->public_dependency_count() > 0) {
printer_->Print(",\npublic_dependencies=[");
for (int i = 0; i < file_->public_dependency_count(); ++i) {
std::string module_alias =
ModuleAlias(file_->public_dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
} else {
printer_->Print("serialized_pb=''\n");
printer_->Print("]");
}
// TODO(falk): Also print options and fix the message_type, enum_type,
@ -467,11 +450,9 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
printer_->Indent();
printer_->Indent();
if (pure_python_workable_) {
for (int i = 0; i < enum_descriptor.value_count(); ++i) {
PrintEnumValueDescriptor(*enum_descriptor.value(i));
printer_->Print(",\n");
}
for (int i = 0; i < enum_descriptor.value_count(); ++i) {
PrintEnumValueDescriptor(*enum_descriptor.value(i));
printer_->Print(",\n");
}
printer_->Outdent();
@ -482,10 +463,8 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
EnumDescriptorProto edp;
printer_->Outdent();
printer_->Print(")\n");
if (pure_python_workable_) {
printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name",
module_level_descriptor_name);
}
printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name",
module_level_descriptor_name);
printer_->Print("\n");
}
@ -535,10 +514,6 @@ void Generator::PrintServiceDescriptor(
void Generator::PrintDescriptorKeyAndModuleName(
const ServiceDescriptor& descriptor) const {
std::string name = ModuleLevelServiceDescriptorName(descriptor);
if (!pure_python_workable_) {
name = "_descriptor.ServiceDescriptor(full_name='" +
descriptor.full_name() + "')";
}
printer_->Print("$descriptor_key$ = $descriptor_name$,\n", "descriptor_key",
kDescriptorKey, "descriptor_name", name);
std::string module_name = ModuleName(file_->name());
@ -728,12 +703,7 @@ void Generator::PrintMessage(const Descriptor& message_descriptor,
PrintNestedMessages(message_descriptor, qualified_name, to_register);
std::map<std::string, std::string> m;
m["descriptor_key"] = kDescriptorKey;
if (pure_python_workable_) {
m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor);
} else {
m["descriptor_name"] = "_descriptor.Descriptor(full_name='" +
message_descriptor.full_name() + "')";
}
m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor);
printer_->Print(m, "'$descriptor_key$' : $descriptor_name$,\n");
std::string module_name = ModuleName(file_->name());
printer_->Print("'__module__' : '$module_name$'\n", "module_name",

@ -174,7 +174,6 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_.
mutable std::string file_descriptor_serialized_;
mutable io::Printer* printer_; // Set in Generate(). Under mutex_.
mutable bool pure_python_workable_;
bool opensource_runtime_ = true;

@ -1577,7 +1577,11 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase {
const FileOptions& options() const;
// Syntax of this file.
enum Syntax {
enum Syntax
#ifndef SWIG
: int
#endif // !SWIG
{
SYNTAX_UNKNOWN = 0,
SYNTAX_PROTO2 = 2,
SYNTAX_PROTO3 = 3,

@ -485,6 +485,11 @@ class PROTOBUF_EXPORT Reflection final {
return internal::ToIntSize(SpaceUsedLong(message));
}
// Returns true if the given message is a default message instance.
bool IsDefaultInstance(const Message& message) const {
return schema_.IsDefaultInstance(message);
}
// Check if the given non-repeated field is set.
bool HasField(const Message& message, const FieldDescriptor* field) const;

@ -1343,5 +1343,13 @@ TEST(MESSAGE_TEST_NAME, TestBoolParsers) {
}
}
TEST(MESSAGE_TEST_NAME, IsDefaultInstance) {
UNITTEST::TestAllTypes msg;
const auto& default_msg = UNITTEST::TestAllTypes::default_instance();
const auto* r = msg.GetReflection();
EXPECT_TRUE(r->IsDefaultInstance(default_msg));
EXPECT_FALSE(r->IsDefaultInstance(msg));
}
} // namespace protobuf
} // namespace google

@ -94,43 +94,30 @@ constexpr int RepeatedFieldLowerClampLimit() {
constexpr int kRepeatedFieldUpperClampLimit =
(std::numeric_limits<int>::max() / 2) + 1;
// Swaps two blocks of memory of size sizeof(T).
template <typename T>
inline void SwapBlock(char* p, char* q) {
T tmp;
memcpy(&tmp, p, sizeof(T));
memcpy(p, q, sizeof(T));
memcpy(q, &tmp, sizeof(T));
}
// Swaps two blocks of memory of size kSize:
// template <int kSize> void memswap(char* p, char* q);
template <int kSize>
inline typename std::enable_if<(kSize == 0), void>::type memswap(char*, char*) {
}
#define PROTO_MEMSWAP_DEF_SIZE(reg_type, max_size) \
template <int kSize> \
typename std::enable_if<(kSize >= sizeof(reg_type) && kSize < (max_size)), \
void>::type \
memswap(char* p, char* q) { \
SwapBlock<reg_type>(p, q); \
memswap<kSize - sizeof(reg_type)>(p + sizeof(reg_type), \
q + sizeof(reg_type)); \
}
PROTO_MEMSWAP_DEF_SIZE(uint8_t, 2)
PROTO_MEMSWAP_DEF_SIZE(uint16_t, 4)
PROTO_MEMSWAP_DEF_SIZE(uint32_t, 8)
#ifdef __SIZEOF_INT128__
PROTO_MEMSWAP_DEF_SIZE(uint64_t, 16)
PROTO_MEMSWAP_DEF_SIZE(__uint128_t, (1u << 31))
template <size_t kSize>
void memswap(char* a, char* b) {
#if __SIZEOF_INT128__
using Buffer = __uint128_t;
#else
PROTO_MEMSWAP_DEF_SIZE(uint64_t, (1u << 31))
using Buffer = uint64_t;
#endif
#undef PROTO_MEMSWAP_DEF_SIZE
constexpr size_t kBlockSize = sizeof(Buffer);
Buffer buf;
for (size_t i = 0; i < kSize / kBlockSize; ++i) {
memcpy(&buf, a, kBlockSize);
memcpy(a, b, kBlockSize);
memcpy(b, &buf, kBlockSize);
a += kBlockSize;
b += kBlockSize;
}
// Swap the leftover bytes, could be zero.
memcpy(&buf, a, kSize % kBlockSize);
memcpy(a, b, kSize % kBlockSize);
memcpy(b, &buf, kSize % kBlockSize);
}
template <typename Element>
class RepeatedIterator;

@ -305,6 +305,60 @@ TEST(RepeatedField, SwapLargeLarge) {
}
}
template <int kSize>
void TestMemswap() {
SCOPED_TRACE(kSize);
const auto a_char = [](int i) -> char { return (i % ('z' - 'a')) + 'a'; };
const auto b_char = [](int i) -> char { return (i % ('Z' - 'A')) + 'A'; };
std::string a, b;
for (int i = 0; i < kSize; ++i) {
a += a_char(i);
b += b_char(i);
}
// We will not swap these.
a += "+";
b += "-";
std::string expected_a = b, expected_b = a;
expected_a.back() = '+';
expected_b.back() = '-';
internal::memswap<kSize>(&a[0], &b[0]);
// ODR use the functions in a way that forces the linker to keep them. That
// way we can see their generated code.
volatile auto odr_use_for_asm_dump = &internal::memswap<kSize>;
(void)odr_use_for_asm_dump;
EXPECT_EQ(expected_a, a);
EXPECT_EQ(expected_b, b);
}
TEST(Memswap, VerifyWithSmallAndLargeSizes) {
// Arbitrary sizes
TestMemswap<0>();
TestMemswap<1>();
TestMemswap<10>();
TestMemswap<100>();
TestMemswap<1000>();
TestMemswap<10000>();
TestMemswap<100000>();
TestMemswap<1000000>();
// Pointer aligned sizes
TestMemswap<sizeof(void*) * 1>();
TestMemswap<sizeof(void*) * 7>();
TestMemswap<sizeof(void*) * 17>();
TestMemswap<sizeof(void*) * 27>();
// Test also just the block size and no leftover.
TestMemswap<64 * 1>();
TestMemswap<64 * 2>();
TestMemswap<64 * 3>();
TestMemswap<64 * 4>();
}
// Determines how much space was reserved by the given field by adding elements
// to it until it re-allocates its space.
static int ReservedSpace(RepeatedField<int>* field) {

Loading…
Cancel
Save