Chiebot-Mirror
/
protobuf
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Sync from Piper @465378928

Browse Source 
PROTOBUF_SYNC_PIPER
pull/10360/head
Matt Kulukundis 3 years ago
parent c7b2991888
commit e8ab276325
25 changed files with 461 additions and 341 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 144
      cmake/extract_includes.bat.in
  2. 6
      csharp/src/Google.Protobuf/JsonFormatter.cs
  3. 1
      docs/third_party.md
  4. 21
      kokoro/linux/cmake_shared/build.sh
  5. 41
      php/ext/google/protobuf/def.c
  6. 20
      php/tests/DescriptorsTest.php
  7. 1
      php/tests/GeneratedClassTest.php
  8. 1
      php/tests/proto/test_descriptors.proto
  9. 2
      php/tests/proto/test_php_namespace.proto
  10. 8
      python/google/protobuf/descriptor.py
  11. 13
      python/google/protobuf/internal/message_test.py
  12. 46
      src/google/protobuf/arenaz_sampler.cc
  13. 35
      src/google/protobuf/arenaz_sampler.h
  14. 147
      src/google/protobuf/arenaz_sampler_test.cc
  15. 3
      src/google/protobuf/compiler/command_line_interface.cc
  16. 2
      src/google/protobuf/compiler/cpp/options.h
  17. 5
      src/google/protobuf/compiler/java/file.cc
  18. 33
      src/google/protobuf/compiler/php/php_generator.cc
  19. 146
      src/google/protobuf/compiler/python/generator.cc
  20. 1
      src/google/protobuf/compiler/python/generator.h
  21. 6
      src/google/protobuf/descriptor.h
  22. 5
      src/google/protobuf/message.h
  23. 8
      src/google/protobuf/message_unittest.inc
  24. 53
      src/google/protobuf/repeated_field.h
  25. 54
      src/google/protobuf/repeated_field_unittest.cc

144
cmake/extract_includes.bat.in
Unescape View File

@ -1,144 +0,0 @@
mkdir include
mkdir include\google
mkdir include\google\protobuf
mkdir include\google\protobuf\compiler
mkdir include\google\protobuf\compiler\cpp
mkdir include\google\protobuf\compiler\csharp
mkdir include\google\protobuf\compiler\java
mkdir include\google\protobuf\compiler\objectivec
mkdir include\google\protobuf\compiler\php
mkdir include\google\protobuf\compiler\python
mkdir include\google\protobuf\compiler\ruby
mkdir include\google\protobuf\io
mkdir include\google\protobuf\stubs
mkdir include\google\protobuf\util
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.h" include\google\protobuf\any.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.pb.h" include\google\protobuf\any.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.pb.h" include\google\protobuf\api.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena.h" include\google\protobuf\arena.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena_impl.h" include\google\protobuf\arena_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenastring.h" include\google\protobuf\arenastring.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenaz_sampler.h" include\google\protobuf\arenaz_sampler.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\code_generator.h" include\google\protobuf\compiler\code_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\command_line_interface.h" include\google\protobuf\compiler\command_line_interface.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\file.h" include\google\protobuf\compiler\cpp\file.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\cpp_generator.h" include\google\protobuf\compiler\cpp\cpp_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\generator.h" include\google\protobuf\compiler\cpp\generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\helpers.h" include\google\protobuf\compiler\cpp\helpers.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\names.h" include\google\protobuf\compiler\cpp\names.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_doc_comment.h" include\google\protobuf\compiler\csharp\csharp_doc_comment.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_generator.h" include\google\protobuf\compiler\csharp\csharp_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_names.h" include\google\protobuf\compiler\csharp\csharp_names.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_options.h" include\google\protobuf\compiler\csharp\csharp_options.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\importer.h" include\google\protobuf\compiler\importer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\generator.h" include\google\protobuf\compiler\java\generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_generator.h" include\google\protobuf\compiler\java\java_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\kotlin_generator.h" include\google\protobuf\compiler\java\kotlin_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\names.h" include\google\protobuf\compiler\java\names.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_generator.h" include\google\protobuf\compiler\objectivec\objectivec_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_helpers.h" include\google\protobuf\compiler\objectivec\objectivec_helpers.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\parser.h" include\google\protobuf\compiler\parser.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\php\php_generator.h" include\google\protobuf\compiler\php\php_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.h" include\google\protobuf\compiler\plugin.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.pb.h" include\google\protobuf\compiler\plugin.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\generator.h" include\google\protobuf\compiler\python\generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\pyi_generator.h" include\google\protobuf\compiler\python\pyi_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\python_generator.h" include\google\protobuf\compiler\python\python_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\ruby\ruby_generator.h" include\google\protobuf\compiler\ruby\ruby_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.h" include\google\protobuf\descriptor.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.pb.h" include\google\protobuf\descriptor.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor_database.h" include\google\protobuf\descriptor_database.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.pb.h" include\google\protobuf\duration.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\dynamic_message.h" include\google\protobuf\dynamic_message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.pb.h" include\google\protobuf\empty.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\endian.h" include\google\protobuf\endian.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\explicitly_constructed.h" include\google\protobuf\explicitly_constructed.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set.h" include\google\protobuf\extension_set.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set_inl.h" include\google\protobuf\extension_set_inl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_access_listener.h" include\google\protobuf\field_access_listener.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.pb.h" include\google\protobuf\field_mask.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_reflection.h" include\google\protobuf\generated_enum_reflection.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_util.h" include\google\protobuf\generated_enum_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_bases.h" include\google\protobuf\generated_message_bases.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_reflection.h" include\google\protobuf\generated_message_reflection.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_decl.h" include\google\protobuf\generated_message_tctable_decl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_impl.h" include\google\protobuf\generated_message_tctable_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\inlined_string_field.h" include\google\protobuf\inlined_string_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\printer.h" include\google\protobuf\io\printer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\strtod.h" include\google\protobuf\io\strtod.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\tokenizer.h" include\google\protobuf\io\tokenizer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream.h" include\google\protobuf\io\zero_copy_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream_impl.h" include\google\protobuf\io\zero_copy_stream_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream_impl_lite.h" include\google\protobuf\io\zero_copy_stream_impl_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map.h" include\google\protobuf\map.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_entry.h" include\google\protobuf\map_entry.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_entry_lite.h" include\google\protobuf\map_entry_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field.h" include\google\protobuf\map_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field_inl.h" include\google\protobuf\map_field_inl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field_lite.h" include\google\protobuf\map_field_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_type_handler.h" include\google\protobuf\map_type_handler.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\message.h" include\google\protobuf\message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\message_lite.h" include\google\protobuf\message_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\metadata.h" include\google\protobuf\metadata.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\metadata_lite.h" include\google\protobuf\metadata_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\parse_context.h" include\google\protobuf\parse_context.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port.h" include\google\protobuf\port.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_def.inc" include\google\protobuf\port_def.inc
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_undef.inc" include\google\protobuf\port_undef.inc
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection.h" include\google\protobuf\reflection.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection_ops.h" include\google\protobuf\reflection_ops.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" include\google\protobuf\repeated_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_ptr_field.h" include\google\protobuf\repeated_ptr_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" include\google\protobuf\stubs\bytestream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\map_util.h" include\google\protobuf\stubs\map_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\mutex.h" include\google\protobuf\stubs\mutex.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\once.h" include\google\protobuf\stubs\once.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\platform_macros.h" include\google\protobuf\stubs\platform_macros.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\port.h" include\google\protobuf\stubs\port.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\status.h" include\google\protobuf\stubs\status.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stl_util.h" include\google\protobuf\stubs\stl_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stringpiece.h" include\google\protobuf\stubs\stringpiece.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\strutil.h" include\google\protobuf\stubs\strutil.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\template_util.h" include\google\protobuf\stubs\template_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\text_format.h" include\google\protobuf\text_format.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\timestamp.pb.h" include\google\protobuf\timestamp.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\type.pb.h" include\google\protobuf\type.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\unknown_field_set.h" include\google\protobuf\unknown_field_set.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\delimited_message_util.h" include\google\protobuf\util\delimited_message_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\field_comparator.h" include\google\protobuf\util\field_comparator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\field_mask_util.h" include\google\protobuf\util\field_mask_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\json_util.h" include\google\protobuf\util\json_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\message_differencer.h" include\google\protobuf\util\message_differencer.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\time_util.h" include\google\protobuf\util\time_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver.h" include\google\protobuf\util\type_resolver.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver_util.h" include\google\protobuf\util\type_resolver_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format.h" include\google\protobuf\wire_format.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite.h" include\google\protobuf\wire_format_lite.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.pb.h" include\google\protobuf\wrappers.pb.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.proto" include\google\protobuf\any.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.proto" include\google\protobuf\api.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.proto" include\google\protobuf\compiler\plugin.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.proto" include\google\protobuf\descriptor.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.proto" include\google\protobuf\duration.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.proto" include\google\protobuf\empty.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.proto" include\google\protobuf\field_mask.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.proto" include\google\protobuf\source_context.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.proto" include\google\protobuf\struct.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\timestamp.proto" include\google\protobuf\timestamp.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\type.proto" include\google\protobuf\type.proto
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.proto" include\google\protobuf\wrappers.proto

6
csharp/src/Google.Protobuf/JsonFormatter.cs
Unescape View File

@ -154,7 +154,7 @@ namespace Google.Protobuf
/// </summary>
/// <param name="message">The message to format.</param>
/// <param name="indentationLevel">Indentation level to start at.</param>
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <see cref="indentationLevel"/>. E.g:
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <paramref name="indentationLevel"/>. E.g:
/// <code>
/// var response = $@"{{
/// ""data"": { Format(message, indentationLevel: 1) }
@ -178,12 +178,12 @@ namespace Google.Protobuf
public void Format(IMessage message, TextWriter writer) => Format(message, writer, indentationLevel: 0);
/// <summary>
/// Formats the specified message as JSON. When <see cref="Settings.Indentation"/> is not null, start indenting at the specified <see cref="indentationLevel"/>.
/// Formats the specified message as JSON. When <see cref="Settings.Indentation"/> is not null, start indenting at the specified <paramref name="indentationLevel"/>.
/// </summary>
/// <param name="message">The message to format.</param>
/// <param name="writer">The TextWriter to write the formatted message to.</param>
/// <param name="indentationLevel">Indentation level to start at.</param>
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <see cref="indentationLevel"/>.</remarks>
/// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <paramref name="indentationLevel"/>.</remarks>
public void Format(IMessage message, TextWriter writer, int indentationLevel)
{
ProtoPreconditions.CheckNotNull(message, nameof(message));

1
docs/third_party.md
Unescape View File

@ -203,3 +203,4 @@ There are miscellaneous other things you may find useful as a Protocol Buffers d
* [vim-protolint: A protobuf linter for Vim](https://github.com/yoheimuta/vim-protolint)
* [super-linter: Protocol Buffer lint as GitHub Action](https://github.com/github/super-linter)
* [protoc-gen-fieldmask - A plugin to generate static type fieldmask paths](https://github.com/idodod/protoc-gen-fieldmask)
* [protoc-gen-bq-schema - A protoc plugin to generate BigQuery schema files](https://github.com/GoogleCloudPlatform/protoc-gen-bq-schema)

21
kokoro/linux/cmake_shared/build.sh
Unescape View File

@ -4,4 +4,23 @@
set -eux
# TODO(mkruskal) Implement this.
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=`pwd`
CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
# Update git submodules
git submodule update --init --recursive
tmpfile=$(mktemp -u)
docker run \
--cidfile $tmpfile \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
/test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_SHARED_LIBS=ON
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

41
php/ext/google/protobuf/def.c
Unescape View File

@ -456,6 +456,44 @@ PHP_METHOD(FieldDescriptor, getEnumType) {
RETURN_COPY_VALUE(&ret);
}
/*
* FieldDescriptor::getContainingOneof()
*
* Returns the OneofDescriptor for this field, or null if it is not inside
* a oneof.
*/
PHP_METHOD(FieldDescriptor, getContainingOneof) {
FieldDescriptor *intern = (FieldDescriptor*)Z_OBJ_P(getThis());
const upb_OneofDef *o = upb_FieldDef_ContainingOneof(intern->fielddef);
zval ret;
if (!o) {
RETURN_NULL();
}
OneofDescriptor_FromOneofDef(&ret, o);
RETURN_COPY_VALUE(&ret);
}
/*
* FieldDescriptor::getRealContainingOneof()
*
* Returns the non-synthetic OneofDescriptor for this field, or null if it is
* not inside a oneof.
*/
PHP_METHOD(FieldDescriptor, getRealContainingOneof) {
FieldDescriptor *intern = (FieldDescriptor*)Z_OBJ_P(getThis());
const upb_OneofDef *o = upb_FieldDef_RealContainingOneof(intern->fielddef);
zval ret;
if (!o) {
RETURN_NULL();
}
OneofDescriptor_FromOneofDef(&ret, o);
RETURN_COPY_VALUE(&ret);
}
/*
* FieldDescriptor::getMessageType()
*
@ -482,6 +520,8 @@ static zend_function_entry FieldDescriptor_methods[] = {
PHP_ME(FieldDescriptor, getType, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, isMap, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getEnumType, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getContainingOneof, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getRealContainingOneof, arginfo_void, ZEND_ACC_PUBLIC)
PHP_ME(FieldDescriptor, getMessageType, arginfo_void, ZEND_ACC_PUBLIC)
ZEND_FE_END
};
@ -516,6 +556,7 @@ static zend_class_entry *Descriptor_GetGeneratedClass(const upb_MessageDef *m) {
char *classname =
GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m), false);
zend_error(E_ERROR, "Couldn't load generated class %s", classname);
return NULL;
}
void Descriptor_FromMessageDef(zval *val, const upb_MessageDef *m) {

20
php/tests/DescriptorsTest.php
Unescape View File

@ -83,10 +83,10 @@ class DescriptorsTest extends TestBase
$this->assertSame($class, $desc->getClass());
$this->assertInstanceOf('\Google\Protobuf\FieldDescriptor', $desc->getField(0));
$this->assertSame(7, $desc->getFieldCount());
$this->assertSame(8, $desc->getFieldCount());
$this->assertInstanceOf('\Google\Protobuf\OneofDescriptor', $desc->getOneofDecl(0));
$this->assertSame(1, $desc->getOneofDeclCount());
$this->assertSame(2, $desc->getOneofDeclCount());
}
public function testDescriptorForIncludedMessage()
@ -180,6 +180,7 @@ class DescriptorsTest extends TestBase
$this->assertSame(self::GPBTYPE_MESSAGE, $fieldDesc->getType());
$this->assertInstanceOf('\Google\Protobuf\Descriptor', $fieldDesc->getMessageType());
$this->assertFalse($fieldDesc->isMap());
$this->assertNull($fieldDesc->getContainingOneof());
// Oneof int field
// Tested further in testOneofDescriptor()
@ -189,6 +190,21 @@ class DescriptorsTest extends TestBase
$this->assertSame(self::GPBLABEL_OPTIONAL, $fieldDesc->getLabel());
$this->assertSame(self::GPBTYPE_INT32, $fieldDesc->getType());
$this->assertFalse($fieldDesc->isMap());
$this->assertSame($fieldDesc->getContainingOneof(), $fieldDesc->getRealContainingOneof());
$oneofDesc = $fieldDesc->getContainingOneof();
$this->assertSame('my_oneof', $oneofDesc->getName());
// Proto3 optional it field.
// Tested further in testOneofDescriptor()
$fieldDesc = $fieldDescMap[52];
$this->assertSame('proto3_optional_int32', $fieldDesc->getName());
$this->assertSame(52, $fieldDesc->getNumber());
$this->assertSame(self::GPBLABEL_OPTIONAL, $fieldDesc->getLabel());
$this->assertSame(self::GPBTYPE_INT32, $fieldDesc->getType());
$this->assertFalse($fieldDesc->isMap());
$this->assertNull($fieldDesc->getRealContainingOneof());
$this->assertNotNull($fieldDesc->getContainingOneof());
// Map int-enum field
$fieldDesc = $fieldDescMap[71];

1
php/tests/GeneratedClassTest.php
Unescape View File

@ -338,6 +338,7 @@ class GeneratedClassTest extends TestBase
{
$this->assertTrue(class_exists('\Upper\READONLY'));
$this->assertTrue(class_exists('\Lower\readonly'));
$this->assertTrue(class_exists('\Php\Test\TestNamespace\PBEmpty\ReadOnly'));
}
public function testLegacyReadOnlyEnum()

1
php/tests/proto/test_descriptors.proto
Unescape View File

@ -14,6 +14,7 @@ message TestDescriptorsMessage {
oneof my_oneof {
int32 oneof_int32 = 51;
}
optional int32 proto3_optional_int32 = 52;
map<int32, EnumSub> map_int32_enum = 71;

2
php/tests/proto/test_php_namespace.proto
Unescape View File

@ -27,5 +27,7 @@ message TestNamespace {
enum NestedEnum {
ZERO = 0;
};
// Test previously unreserved name
message ReadOnly {}
}
}

8
python/google/protobuf/descriptor.py
Unescape View File

@ -1021,13 +1021,7 @@ class FileDescriptor(DescriptorBase):
# FileDescriptor() is called from various places, not only from generated
# files, to register dynamic proto files and messages.
# pylint: disable=g-explicit-bool-comparison
if serialized_pb == b'':
# Cpp generated code must be linked in if serialized_pb is ''
try:
return _message.default_pool.FindFileByName(name)
except KeyError:
raise RuntimeError('Please link in cpp generated lib for %s' % (name))
elif serialized_pb:
if serialized_pb:
return _message.default_pool.AddSerializedFile(serialized_pb)
else:
return super(FileDescriptor, cls).__new__(cls)

13
python/google/protobuf/internal/message_test.py
Unescape View File

@ -1768,6 +1768,19 @@ class Proto3Test(unittest.TestCase):
with self.assertRaises(TypeError):
123 in msg.map_string_string
def testScalarMapComparison(self):
msg1 = map_unittest_pb2.TestMap()
msg2 = map_unittest_pb2.TestMap()
self.assertEqual(msg1.map_int32_int32, msg2.map_int32_int32)
def testMessageMapComparison(self):
msg1 = map_unittest_pb2.TestMap()
msg2 = map_unittest_pb2.TestMap()
self.assertEqual(msg1.map_int32_foreign_message,
msg2.map_int32_foreign_message)
def testMapGet(self):
# Need to test that get() properly returns the default, even though the dict
# has defaultdict-like semantics.

46
src/google/protobuf/arenaz_sampler.cc
Unescape View File

@ -33,6 +33,7 @@
#include <atomic>
#include <cstdint>
#include <limits>
#include <utility>
// Must be included last.
@ -74,11 +75,15 @@ PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = {
ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); }
ThreadSafeArenaStats::~ThreadSafeArenaStats() = default;
void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
void ThreadSafeArenaStats::BlockStats::PrepareForSampling() {
num_allocations.store(0, std::memory_order_relaxed);
bytes_used.store(0, std::memory_order_relaxed);
bytes_allocated.store(0, std::memory_order_relaxed);
bytes_used.store(0, std::memory_order_relaxed);
bytes_wasted.store(0, std::memory_order_relaxed);
}
void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
for (auto& blockstats : block_histogram) blockstats.PrepareForSampling();
max_block_size.store(0, std::memory_order_relaxed);
thread_ids.store(0, std::memory_order_relaxed);
weight = stride;
@ -88,12 +93,41 @@ void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
depth = absl::GetStackTrace(stack, kMaxStackDepth, /* skip_count= */ 0);
}
size_t ThreadSafeArenaStats::FindBin(size_t bytes) {
if (bytes <= kMaxSizeForBinZero) return 0;
if (bytes <= kMaxSizeForPenultimateBin) {
// absl::bit_width() returns one plus the base-2 logarithm of x, with any
// fractional part discarded.
return absl::bit_width(absl::bit_ceil(bytes)) - kLogMaxSizeForBinZero - 1;
}
return kBlockHistogramBins - 1;
}
std::pair<size_t, size_t> ThreadSafeArenaStats::MinMaxBlockSizeForBin(
size_t bin) {
ABSL_ASSERT(bin < kBlockHistogramBins);
if (bin == 0) return {1, kMaxSizeForBinZero};
if (bin < kBlockHistogramBins - 1) {
return {(1 << (kLogMaxSizeForBinZero + bin - 1)) + 1,
1 << (kLogMaxSizeForBinZero + bin)};
}
return {kMaxSizeForPenultimateBin + 1, std::numeric_limits<size_t>::max()};
}
void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t used,
size_t allocated, size_t wasted) {
info->num_allocations.fetch_add(1, std::memory_order_relaxed);
info->bytes_used.fetch_add(used, std::memory_order_relaxed);
info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
// Update the allocated bytes for the current block.
ThreadSafeArenaStats::BlockStats& curr =
info->block_histogram[ThreadSafeArenaStats::FindBin(allocated)];
curr.bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
curr.num_allocations.fetch_add(1, std::memory_order_relaxed);
// Update the used and wasted bytes for the previous block.
ThreadSafeArenaStats::BlockStats& prev =
info->block_histogram[ThreadSafeArenaStats::FindBin(used + wasted)];
prev.bytes_used.fetch_add(used, std::memory_order_relaxed);
prev.bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
if (info->max_block_size.load(std::memory_order_relaxed) < allocated) {
info->max_block_size.store(allocated, std::memory_order_relaxed);
}

35
src/google/protobuf/arenaz_sampler.h
Unescape View File

@ -31,9 +31,11 @@
#ifndef GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__
#define GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__
#include <array>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <utility>
// Must be included last.
@ -65,10 +67,28 @@ struct ThreadSafeArenaStats
// These fields are mutated by the various Record* APIs and need to be
// thread-safe.
std::atomic<int> num_allocations;
std::atomic<size_t> bytes_used;
std::atomic<size_t> bytes_allocated;
std::atomic<size_t> bytes_wasted;
struct BlockStats {
std::atomic<int> num_allocations;
std::atomic<size_t> bytes_allocated;
std::atomic<size_t> bytes_used;
std::atomic<size_t> bytes_wasted;
void PrepareForSampling();
};
// block_histogram is a kBlockHistogramBins sized histogram. The zeroth bin
// stores info about blocks of size \in [1, 1 << kLogMaxSizeForBinZero]. Bin
// i, where i > 0, stores info for blocks of size \in (max_size_bin (i-1),
// 1 << (kLogMaxSizeForBinZero + i)]. The final bin stores info about blocks
// of size \in [kMaxSizeForPenultimateBin + 1,
// std::numeric_limits<size_t>::max()].
static constexpr size_t kBlockHistogramBins = 15;
static constexpr size_t kLogMaxSizeForBinZero = 7;
static constexpr size_t kMaxSizeForBinZero = (1 << kLogMaxSizeForBinZero);
static constexpr size_t kMaxSizeForPenultimateBin =
1 << (kLogMaxSizeForBinZero + kBlockHistogramBins - 2);
std::array<BlockStats, kBlockHistogramBins> block_histogram;
// Records the largest block allocated for the arena.
std::atomic<size_t> max_block_size;
// Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed
@ -90,6 +110,13 @@ struct ThreadSafeArenaStats
if (PROTOBUF_PREDICT_TRUE(info == nullptr)) return;
RecordAllocateSlow(info, used, allocated, wasted);
}
// Returns the bin for the provided size.
static size_t FindBin(size_t bytes);
// Returns the min and max bytes that can be stored in the histogram for
// blocks in the provided bin.
static std::pair<size_t, size_t> MinMaxBlockSizeForBin(size_t bin);
};
struct SamplingState {

147
src/google/protobuf/arenaz_sampler_test.cc
Unescape View File

@ -30,6 +30,8 @@
#include <google/protobuf/arenaz_sampler.h>
#include <atomic>
#include <limits>
#include <memory>
#include <random>
#include <utility>
@ -37,7 +39,6 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/strutil.h>
// Must be included last.
@ -57,10 +58,17 @@ class ThreadSafeArenaStatsHandlePeer {
return h->info_;
}
};
std::vector<size_t> GetBytesAllocated(ThreadSafeArenazSampler* s) {
std::vector<size_t> res;
s->Iterate([&](const ThreadSafeArenaStats& info) {
res.push_back(info.bytes_allocated.load(std::memory_order_acquire));
for (const auto& block_stats : info.block_histogram) {
size_t bytes_allocated =
block_stats.bytes_allocated.load(std::memory_order_acquire);
if (bytes_allocated != 0) {
res.push_back(bytes_allocated);
}
}
});
return res;
}
@ -69,7 +77,8 @@ ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size,
int64_t stride) {
auto* info = s->Register(stride);
assert(info != nullptr);
info->bytes_allocated.store(size);
info->block_histogram[0].bytes_allocated.store(size,
std::memory_order_relaxed);
return info;
}
@ -85,46 +94,100 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) {
MutexLock l(&info.init_mu);
info.PrepareForSampling(kTestStride);
EXPECT_EQ(info.num_allocations.load(), 0);
EXPECT_EQ(info.bytes_used.load(), 0);
EXPECT_EQ(info.bytes_allocated.load(), 0);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_block_size.load(), 0);
for (const auto& block_stats : info.block_histogram) {
EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0);
}
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0);
EXPECT_EQ(info.weight, kTestStride);
info.num_allocations.store(1, std::memory_order_relaxed);
info.bytes_used.store(1, std::memory_order_relaxed);
info.bytes_allocated.store(1, std::memory_order_relaxed);
info.bytes_wasted.store(1, std::memory_order_relaxed);
for (auto& block_stats : info.block_histogram) {
block_stats.num_allocations.store(1, std::memory_order_relaxed);
block_stats.bytes_used.store(1, std::memory_order_relaxed);
block_stats.bytes_allocated.store(1, std::memory_order_relaxed);
block_stats.bytes_wasted.store(1, std::memory_order_relaxed);
}
info.max_block_size.store(1, std::memory_order_relaxed);
info.PrepareForSampling(2 * kTestStride);
EXPECT_EQ(info.num_allocations.load(), 0);
EXPECT_EQ(info.bytes_used.load(), 0);
EXPECT_EQ(info.bytes_allocated.load(), 0);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_block_size.load(), 0);
for (auto& block_stats : info.block_histogram) {
EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0);
EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0);
}
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0);
EXPECT_EQ(info.weight, 2 * kTestStride);
}
TEST(ThreadSafeArenaStatsTest, FindBin) {
size_t current_bin = 0;
size_t bytes = 1;
while (current_bin < ThreadSafeArenaStats::kBlockHistogramBins - 1) {
size_t next_bin = ThreadSafeArenaStats::FindBin(bytes);
if (next_bin != current_bin) {
// Test the bins increase linearly.
EXPECT_EQ(next_bin, current_bin + 1);
// Test the bins change only at values of the form 2^k + 1.
EXPECT_EQ(absl::popcount(bytes - 1), 1);
current_bin = next_bin;
}
++bytes;
}
}
TEST(ThreadSafeArenaStatsTest, MinMaxBlockSizeForBin) {
std::pair<size_t, size_t> current_limits =
ThreadSafeArenaStats::MinMaxBlockSizeForBin(0);
EXPECT_EQ(current_limits.first, 1);
EXPECT_LT(current_limits.first, current_limits.second);
for (size_t i = 1; i < ThreadSafeArenaStats::kBlockHistogramBins; ++i) {
std::pair<size_t, size_t> next_limits =
ThreadSafeArenaStats::MinMaxBlockSizeForBin(i);
EXPECT_LT(next_limits.first, next_limits.second);
// Test the limits do not have gaps.
EXPECT_EQ(next_limits.first, current_limits.second + 1);
if (i != ThreadSafeArenaStats::kBlockHistogramBins - 1) {
EXPECT_EQ(next_limits.second, 2 * current_limits.second);
}
current_limits = next_limits;
}
// Test the limits cover the entire range possible.
EXPECT_EQ(current_limits.second, std::numeric_limits<size_t>::max());
}
TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) {
ThreadSafeArenaStats info;
constexpr int64_t kTestStride = 458;
MutexLock l(&info.init_mu);
info.PrepareForSampling(kTestStride);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(info.num_allocations.load(), 1);
EXPECT_EQ(info.bytes_used.load(), 100);
EXPECT_EQ(info.bytes_allocated.load(), 128);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_block_size.load(), 128);
RecordAllocateSlow(&info, /*requested=*/0, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(
info.block_histogram[0].num_allocations.load(std::memory_order_relaxed),
1);
EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed),
0);
EXPECT_EQ(
info.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed),
128);
EXPECT_EQ(
info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 0);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256,
/*wasted=*/28);
EXPECT_EQ(info.num_allocations.load(), 2);
EXPECT_EQ(info.bytes_used.load(), 200);
EXPECT_EQ(info.bytes_allocated.load(), 384);
EXPECT_EQ(info.bytes_wasted.load(), 28);
EXPECT_EQ(info.max_block_size.load(), 256);
EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed),
100);
EXPECT_EQ(
info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 28);
EXPECT_EQ(
info.block_histogram[1].num_allocations.load(std::memory_order_relaxed),
1);
EXPECT_EQ(
info.block_histogram[1].bytes_allocated.load(std::memory_order_relaxed),
256);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256);
}
TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) {
@ -133,13 +196,13 @@ TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) {
MutexLock l(&info.init_mu);
info.PrepareForSampling(kTestStride);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(info.max_block_size.load(), 128);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256,
/*wasted=*/28);
EXPECT_EQ(info.max_block_size.load(), 256);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128,
/*wasted=*/28);
EXPECT_EQ(info.max_block_size.load(), 256);
EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256);
}
TEST(ThreadSafeArenazSamplerTest, SamplingCorrectness) {
@ -212,12 +275,15 @@ TEST(ThreadSafeArenazSamplerTest, Handle) {
constexpr int64_t kTestStride = 17;
ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride));
auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h);
info->bytes_allocated.store(0x12345678, std::memory_order_relaxed);
info->block_histogram[0].bytes_allocated.store(0x12345678,
std::memory_order_relaxed);
bool found = false;
sampler.Iterate([&](const ThreadSafeArenaStats& h) {
if (&h == info) {
EXPECT_EQ(h.bytes_allocated.load(), 0x12345678);
EXPECT_EQ(
h.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed),
0x12345678);
EXPECT_EQ(h.weight, kTestStride);
found = true;
}
@ -230,7 +296,8 @@ TEST(ThreadSafeArenazSamplerTest, Handle) {
if (&h == info) {
// this will only happen if some other thread has resurrected the info
// the old handle was using.
if (h.bytes_allocated.load() == 0x12345678) {
if (h.block_histogram[0].bytes_allocated.load(
std::memory_order_relaxed) == 0x12345678) {
found = true;
}
}
@ -246,7 +313,7 @@ TEST(ThreadSafeArenazSamplerTest, Registration) {
auto* info2 = Register(&sampler, 2, kTestStride);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2));
info1->bytes_allocated.store(3);
info1->block_histogram[0].bytes_allocated.store(3, std::memory_order_relaxed);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2));
sampler.Unregister(info1);
@ -258,18 +325,18 @@ TEST(ThreadSafeArenazSamplerTest, Unregistration) {
std::vector<ThreadSafeArenaStats*> infos;
constexpr int64_t kTestStride = 200;
for (size_t i = 0; i < 3; ++i) {
infos.push_back(Register(&sampler, i, kTestStride));
infos.push_back(Register(&sampler, i + 1, kTestStride));
}
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2, 3));
sampler.Unregister(infos[1]);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3));
infos.push_back(Register(&sampler, 3, kTestStride));
infos.push_back(Register(&sampler, 4, kTestStride));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 3, 4));
sampler.Unregister(infos[3]);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 4));
sampler.Unregister(infos[0]);
sampler.Unregister(infos[2]);

3
src/google/protobuf/compiler/command_line_interface.cc
Unescape View File

@ -2124,7 +2124,8 @@ bool CommandLineInterface::EnforceProto3OptionalSupport(
<< codegen_name
<< " hasn't been updated to support optional fields in "
"proto3. Please ask the owner of this code generator to "
"support proto3 optional.";
"support proto3 optional."
<< std::endl;
return false;
}
}

2
src/google/protobuf/compiler/cpp/options.h
Unescape View File

@ -85,7 +85,7 @@ struct Options {
bool profile_driven_inline_string = true;
bool message_owned_arena_trial = false;
bool force_split = false;
bool profile_driven_split = false;
bool profile_driven_split = true;
#ifdef PROTOBUF_STABLE_EXPERIMENTS
bool force_eagerly_verified_lazy = true;
bool force_inline_string = true;

5
src/google/protobuf/compiler/java/file.cc
Unescape View File

@ -704,10 +704,13 @@ void FileGenerator::GenerateKotlinSiblings(
options_.annotate_code ? &annotation_collector : nullptr);
printer.Print(
"//Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", descriptor->file()->name());
printer.Print(
"// Generated files should ignore deprecation warnings\n"
"@file:Suppress(\"DEPRECATION\")");
if (!java_package_.empty()) {
printer.Print(
"package $package$;\n"

33
src/google/protobuf/compiler/php/php_generator.cc
Unescape View File

@ -421,12 +421,17 @@ std::string LegacyGeneratedClassFileName(const DescriptorType* desc,
}
template <typename DescriptorType>
std::string LegacyReadOnlyGeneratedClassFileName(const DescriptorType* desc,
const Options& options) {
std::string php_namespace = RootPhpNamespace(desc, options);
std::string LegacyReadOnlyGeneratedClassFileName(std::string php_namespace,
const DescriptorType* desc) {
if (!php_namespace.empty()) {
for (int i = 0; i < php_namespace.size(); i++) {
if (php_namespace[i] == '\\') {
php_namespace[i] = '/';
}
}
return php_namespace + "/" + desc->name() + ".php";
}
return desc->name() + ".php";
}
@ -1316,27 +1321,39 @@ template <typename DescriptorType>
void LegacyReadOnlyGenerateClassFile(const FileDescriptor* file,
const DescriptorType* desc, const Options& options,
GeneratorContext* generator_context) {
std::string filename = LegacyReadOnlyGeneratedClassFileName(desc, options);
std::string fullname = FullClassName(desc, options);
std::string php_namespace;
std::string classname;
int lastindex = fullname.find_last_of("\\");
if (lastindex != std::string::npos) {
php_namespace = fullname.substr(0, lastindex);
classname = fullname.substr(lastindex + 1);
} else {
php_namespace = "";
classname = fullname;
}
std::string filename = LegacyReadOnlyGeneratedClassFileName(php_namespace, desc);
std::unique_ptr<io::ZeroCopyOutputStream> output(
generator_context->Open(filename));
io::Printer printer(output.get(), '^');
GenerateHead(file, &printer);
std::string php_namespace = RootPhpNamespace(desc, options);
if (!php_namespace.empty()) {
printer.Print(
"namespace ^name^;\n\n",
"name", php_namespace);
}
std::string newname = FullClassName(desc, options);
printer.Print("class_exists(^new^::class); // autoload the new class, which "
"will also create an alias to the deprecated class\n",
"new", GeneratedClassNameImpl(desc));
"new", classname);
printer.Print("@trigger_error(__NAMESPACE__ . '\\^old^ is deprecated and will be removed in "
"the next major release. Use ^fullname^ instead', E_USER_DEPRECATED);\n\n",
"old", desc->name(),
"fullname", newname);
"fullname", classname);
}
void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en,

146
src/google/protobuf/compiler/python/generator.cc
Unescape View File

@ -217,15 +217,12 @@ bool Generator::Generate(const FileDescriptor* file,
GeneratorContext* context, std::string* error) const {
// -----------------------------------------------------------------
// parse generator options
bool cpp_generated_lib_linked = false;
std::vector<std::pair<std::string, std::string> > options;
ParseGeneratorParameter(parameter, &options);
for (int i = 0; i < options.size(); i++) {
if (options[i].first == "cpp_generated_lib_linked") {
cpp_generated_lib_linked = true;
} else if (options[i].first == "pyi_out") {
if (options[i].first == "pyi_out") {
python::PyiGenerator pyi_generator;
if (!pyi_generator.Generate(file, "", context, error)) {
return false;
@ -247,10 +244,6 @@ bool Generator::Generate(const FileDescriptor* file,
file_ = file;
std::string filename = GetFileName(file, ".py");
pure_python_workable_ = !cpp_generated_lib_linked;
if (HasPrefixString(file->name(), "google/protobuf/")) {
pure_python_workable_ = true;
}
FileDescriptorProto fdp;
file_->CopyTo(&fdp);
@ -263,55 +256,49 @@ bool Generator::Generate(const FileDescriptor* file,
printer_ = &printer;
PrintTopBoilerplate(printer_, file_, GeneratingDescriptorProto());
if (pure_python_workable_) {
PrintImports();
}
PrintImports();
PrintFileDescriptor();
if (pure_python_workable_) {
if (GeneratingDescriptorProto()) {
printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// Create enums before message descriptors
PrintAllNestedEnumsInFile();
PrintMessageDescriptors();
FixForeignFieldsInDescriptors();
printer_->Outdent();
printer_->Print("else:\n");
printer_->Indent();
}
// Find the message descriptors first and then use the message
// descriptor to find enums.
printer_->Print(
"_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n");
if (GeneratingDescriptorProto()) {
printer_->Outdent();
}
if (GeneratingDescriptorProto()) {
printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// Create enums before message descriptors
PrintAllNestedEnumsInFile();
PrintMessageDescriptors();
FixForeignFieldsInDescriptors();
printer_->Outdent();
printer_->Print("else:\n");
printer_->Indent();
}
// Find the message descriptors first and then use the message
// descriptor to find enums.
printer_->Print(
"_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n");
if (GeneratingDescriptorProto()) {
printer_->Outdent();
}
std::string module_name = ModuleName(file->name());
printer_->Print(
"_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '$module_name$', "
"globals())\n",
"module_name", module_name);
if (pure_python_workable_) {
printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// We have to fix up the extensions after the message classes themselves,
// since they need to call static RegisterExtension() methods on these
// classes.
FixForeignFieldsInExtensions();
// Descriptor options may have custom extensions. These custom options
// can only be successfully parsed after we register corresponding
// extensions. Therefore we parse all options again here to recognize
// custom options that may be unknown when we define the descriptors.
// This does not apply to services because they are not used by extensions.
FixAllDescriptorOptions();
// We have to fix up the extensions after the message classes themselves,
// since they need to call static RegisterExtension() methods on these
// classes.
FixForeignFieldsInExtensions();
// Descriptor options may have custom extensions. These custom options
// can only be successfully parsed after we register corresponding
// extensions. Therefore we parse all options again here to recognize
// custom options that may be unknown when we define the descriptors.
// This does not apply to services because they are not used by extensions.
FixAllDescriptorOptions();
// Set serialized_start and serialized_end.
SetSerializedPbInterval();
// Set serialized_start and serialized_end.
SetSerializedPbInterval();
printer_->Outdent();
}
printer_->Outdent();
if (HasGenericServices(file)) {
printer_->Print(
"_builder.BuildServices(DESCRIPTOR, '$module_name$', globals())\n",
@ -389,30 +376,26 @@ void Generator::PrintFileDescriptor() const {
" create_key=_descriptor._internal_create_key,\n";
printer_->Print(m, file_descriptor_template);
printer_->Indent();
if (pure_python_workable_) {
printer_->Print("serialized_pb=b'$value$'\n", "value",
strings::CHexEscape(file_descriptor_serialized_));
if (file_->dependency_count() != 0) {
printer_->Print(",\ndependencies=[");
for (int i = 0; i < file_->dependency_count(); ++i) {
std::string module_alias = ModuleAlias(file_->dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
printer_->Print("]");
printer_->Print("serialized_pb=b'$value$'\n", "value",
strings::CHexEscape(file_descriptor_serialized_));
if (file_->dependency_count() != 0) {
printer_->Print(",\ndependencies=[");
for (int i = 0; i < file_->dependency_count(); ++i) {
std::string module_alias = ModuleAlias(file_->dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
if (file_->public_dependency_count() > 0) {
printer_->Print(",\npublic_dependencies=[");
for (int i = 0; i < file_->public_dependency_count(); ++i) {
std::string module_alias =
ModuleAlias(file_->public_dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
printer_->Print("]");
printer_->Print("]");
}
if (file_->public_dependency_count() > 0) {
printer_->Print(",\npublic_dependencies=[");
for (int i = 0; i < file_->public_dependency_count(); ++i) {
std::string module_alias =
ModuleAlias(file_->public_dependency(i)->name());
printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias",
module_alias);
}
} else {
printer_->Print("serialized_pb=''\n");
printer_->Print("]");
}
// TODO(falk): Also print options and fix the message_type, enum_type,
@ -467,11 +450,9 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
printer_->Indent();
printer_->Indent();
if (pure_python_workable_) {
for (int i = 0; i < enum_descriptor.value_count(); ++i) {
PrintEnumValueDescriptor(*enum_descriptor.value(i));
printer_->Print(",\n");
}
for (int i = 0; i < enum_descriptor.value_count(); ++i) {
PrintEnumValueDescriptor(*enum_descriptor.value(i));
printer_->Print(",\n");
}
printer_->Outdent();
@ -482,10 +463,8 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
EnumDescriptorProto edp;
printer_->Outdent();
printer_->Print(")\n");
if (pure_python_workable_) {
printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name",
module_level_descriptor_name);
}
printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name",
module_level_descriptor_name);
printer_->Print("\n");
}
@ -535,10 +514,6 @@ void Generator::PrintServiceDescriptor(
void Generator::PrintDescriptorKeyAndModuleName(
const ServiceDescriptor& descriptor) const {
std::string name = ModuleLevelServiceDescriptorName(descriptor);
if (!pure_python_workable_) {
name = "_descriptor.ServiceDescriptor(full_name='" +
descriptor.full_name() + "')";
}
printer_->Print("$descriptor_key$ = $descriptor_name$,\n", "descriptor_key",
kDescriptorKey, "descriptor_name", name);
std::string module_name = ModuleName(file_->name());
@ -728,12 +703,7 @@ void Generator::PrintMessage(const Descriptor& message_descriptor,
PrintNestedMessages(message_descriptor, qualified_name, to_register);
std::map<std::string, std::string> m;
m["descriptor_key"] = kDescriptorKey;
if (pure_python_workable_) {
m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor);
} else {
m["descriptor_name"] = "_descriptor.Descriptor(full_name='" +
message_descriptor.full_name() + "')";
}
m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor);
printer_->Print(m, "'$descriptor_key$' : $descriptor_name$,\n");
std::string module_name = ModuleName(file_->name());
printer_->Print("'__module__' : '$module_name$'\n", "module_name",

1
src/google/protobuf/compiler/python/generator.h
Unescape View File

@ -174,7 +174,6 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_.
mutable std::string file_descriptor_serialized_;
mutable io::Printer* printer_; // Set in Generate(). Under mutex_.
mutable bool pure_python_workable_;
bool opensource_runtime_ = true;

6
src/google/protobuf/descriptor.h
Unescape View File

@ -1577,7 +1577,11 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase {
const FileOptions& options() const;
// Syntax of this file.
enum Syntax {
enum Syntax
#ifndef SWIG
: int
#endif // !SWIG
{
SYNTAX_UNKNOWN = 0,
SYNTAX_PROTO2 = 2,
SYNTAX_PROTO3 = 3,

5
src/google/protobuf/message.h
Unescape View File

@ -485,6 +485,11 @@ class PROTOBUF_EXPORT Reflection final {
return internal::ToIntSize(SpaceUsedLong(message));
}
// Returns true if the given message is a default message instance.
bool IsDefaultInstance(const Message& message) const {
return schema_.IsDefaultInstance(message);
}
// Check if the given non-repeated field is set.
bool HasField(const Message& message, const FieldDescriptor* field) const;

8
src/google/protobuf/message_unittest.inc
Unescape View File

@ -1343,5 +1343,13 @@ TEST(MESSAGE_TEST_NAME, TestBoolParsers) {
}
}
TEST(MESSAGE_TEST_NAME, IsDefaultInstance) {
UNITTEST::TestAllTypes msg;
const auto& default_msg = UNITTEST::TestAllTypes::default_instance();
const auto* r = msg.GetReflection();
EXPECT_TRUE(r->IsDefaultInstance(default_msg));
EXPECT_FALSE(r->IsDefaultInstance(msg));
}
} // namespace protobuf
} // namespace google

53
src/google/protobuf/repeated_field.h
Unescape View File

@ -94,43 +94,30 @@ constexpr int RepeatedFieldLowerClampLimit() {
constexpr int kRepeatedFieldUpperClampLimit =
(std::numeric_limits<int>::max() / 2) + 1;
// Swaps two blocks of memory of size sizeof(T).
template <typename T>
inline void SwapBlock(char* p, char* q) {
T tmp;
memcpy(&tmp, p, sizeof(T));
memcpy(p, q, sizeof(T));
memcpy(q, &tmp, sizeof(T));
}
// Swaps two blocks of memory of size kSize:
// template <int kSize> void memswap(char* p, char* q);
template <int kSize>
inline typename std::enable_if<(kSize == 0), void>::type memswap(char*, char*) {
}
#define PROTO_MEMSWAP_DEF_SIZE(reg_type, max_size) \
template <int kSize> \
typename std::enable_if<(kSize >= sizeof(reg_type) && kSize < (max_size)), \
void>::type \
memswap(char* p, char* q) { \
SwapBlock<reg_type>(p, q); \
memswap<kSize - sizeof(reg_type)>(p + sizeof(reg_type), \
q + sizeof(reg_type)); \
}
PROTO_MEMSWAP_DEF_SIZE(uint8_t, 2)
PROTO_MEMSWAP_DEF_SIZE(uint16_t, 4)
PROTO_MEMSWAP_DEF_SIZE(uint32_t, 8)
#ifdef __SIZEOF_INT128__
PROTO_MEMSWAP_DEF_SIZE(uint64_t, 16)
PROTO_MEMSWAP_DEF_SIZE(__uint128_t, (1u << 31))
template <size_t kSize>
void memswap(char* a, char* b) {
#if __SIZEOF_INT128__
using Buffer = __uint128_t;
#else
PROTO_MEMSWAP_DEF_SIZE(uint64_t, (1u << 31))
using Buffer = uint64_t;
#endif
#undef PROTO_MEMSWAP_DEF_SIZE
constexpr size_t kBlockSize = sizeof(Buffer);
Buffer buf;
for (size_t i = 0; i < kSize / kBlockSize; ++i) {
memcpy(&buf, a, kBlockSize);
memcpy(a, b, kBlockSize);
memcpy(b, &buf, kBlockSize);
a += kBlockSize;
b += kBlockSize;
}
// Swap the leftover bytes, could be zero.
memcpy(&buf, a, kSize % kBlockSize);
memcpy(a, b, kSize % kBlockSize);
memcpy(b, &buf, kSize % kBlockSize);
}
template <typename Element>
class RepeatedIterator;

54
src/google/protobuf/repeated_field_unittest.cc
Unescape View File

@ -305,6 +305,60 @@ TEST(RepeatedField, SwapLargeLarge) {
}
}
template <int kSize>
void TestMemswap() {
SCOPED_TRACE(kSize);
const auto a_char = [](int i) -> char { return (i % ('z' - 'a')) + 'a'; };
const auto b_char = [](int i) -> char { return (i % ('Z' - 'A')) + 'A'; };
std::string a, b;
for (int i = 0; i < kSize; ++i) {
a += a_char(i);
b += b_char(i);
}
// We will not swap these.
a += "+";
b += "-";
std::string expected_a = b, expected_b = a;
expected_a.back() = '+';
expected_b.back() = '-';
internal::memswap<kSize>(&a[0], &b[0]);
// ODR use the functions in a way that forces the linker to keep them. That
// way we can see their generated code.
volatile auto odr_use_for_asm_dump = &internal::memswap<kSize>;
(void)odr_use_for_asm_dump;
EXPECT_EQ(expected_a, a);
EXPECT_EQ(expected_b, b);
}
TEST(Memswap, VerifyWithSmallAndLargeSizes) {
// Arbitrary sizes
TestMemswap<0>();
TestMemswap<1>();
TestMemswap<10>();
TestMemswap<100>();
TestMemswap<1000>();
TestMemswap<10000>();
TestMemswap<100000>();
TestMemswap<1000000>();
// Pointer aligned sizes
TestMemswap<sizeof(void*) * 1>();
TestMemswap<sizeof(void*) * 7>();
TestMemswap<sizeof(void*) * 17>();
TestMemswap<sizeof(void*) * 27>();
// Test also just the block size and no leftover.
TestMemswap<64 * 1>();
TestMemswap<64 * 2>();
TestMemswap<64 * 3>();
TestMemswap<64 * 4>();
}
// Determines how much space was reserved by the given field by adding elements
// to it until it re-allocates its space.
static int ReservedSpace(RepeatedField<int>* field) {

Write Preview
Loading…
Cancel
Save