Merge branch 'master' into php-wkt

pull/7944/head
Joshua Haberman 4 years ago
commit a1486fc89d
  1. 1
      BUILD
  2. 17
      CHANGES.txt
  3. 2
      cmake/extract_includes.bat.in
  4. 1
      cmake/libprotobuf-lite.cmake
  5. 12
      cmake/protobuf-module.cmake.in
  6. 20
      conformance/Makefile.am
  7. 1
      conformance/failure_list_js.txt
  8. 78
      conformance/text_format_conformance_suite.cc
  9. 20
      conformance/text_format_failure_list_cpp.txt
  10. 7
      conformance/text_format_failure_list_java.txt
  11. 29
      conformance/text_format_failure_list_python.txt
  12. 36
      conformance/text_format_failure_list_python_2.7.txt
  13. 28
      conformance/text_format_failure_list_python_cpp.txt
  14. 30
      conformance/text_format_failure_list_python_cpp_2.7.txt
  15. 12
      csharp/src/Google.Protobuf.Test/ByteStringTest.cs
  16. 1
      csharp/src/Google.Protobuf/ByteString.cs
  17. 8
      editors/proto.vim
  18. 7
      java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java
  19. 14
      java/core/src/main/java/com/google/protobuf/BooleanArrayList.java
  20. 17
      java/core/src/main/java/com/google/protobuf/CodedInputStream.java
  21. 5
      java/core/src/main/java/com/google/protobuf/Descriptors.java
  22. 14
      java/core/src/main/java/com/google/protobuf/DoubleArrayList.java
  23. 2
      java/core/src/main/java/com/google/protobuf/ExtensionRegistryFactory.java
  24. 2
      java/core/src/main/java/com/google/protobuf/ExtensionRegistryLite.java
  25. 2
      java/core/src/main/java/com/google/protobuf/FieldType.java
  26. 14
      java/core/src/main/java/com/google/protobuf/FloatArrayList.java
  27. 14
      java/core/src/main/java/com/google/protobuf/IntArrayList.java
  28. 1
      java/core/src/main/java/com/google/protobuf/Internal.java
  29. 14
      java/core/src/main/java/com/google/protobuf/LongArrayList.java
  30. 2
      java/core/src/main/java/com/google/protobuf/MessageSchema.java
  31. 7
      java/core/src/main/java/com/google/protobuf/NioByteString.java
  32. 7
      java/core/src/main/java/com/google/protobuf/Protobuf.java
  33. 14
      java/core/src/main/java/com/google/protobuf/RopeByteString.java
  34. 69
      java/core/src/main/java/com/google/protobuf/TextFormat.java
  35. 1
      java/core/src/main/java/com/google/protobuf/TypeRegistry.java
  36. 13
      java/core/src/main/java/com/google/protobuf/UnsafeUtil.java
  37. 14
      java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java
  38. 165
      java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java
  39. 108
      java/core/src/test/java/com/google/protobuf/MapForProto2Test.java
  40. 179
      java/core/src/test/java/com/google/protobuf/MapLiteTest.java
  41. 190
      java/core/src/test/java/com/google/protobuf/MapTest.java
  42. 88
      java/core/src/test/java/com/google/protobuf/TextFormatTest.java
  43. 5
      java/lite/src/test/java/com/google/protobuf/Proto2MessageLiteInfoFactory.java
  44. 60
      java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java
  45. 8
      java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java
  46. 1
      java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
  47. 24
      java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java
  48. 2
      java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java
  49. 4
      java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
  50. 1
      js/binary/constants.js
  51. 1
      js/binary/decoder.js
  52. 1
      js/binary/reader.js
  53. 1
      js/binary/utils.js
  54. 1
      js/binary/writer.js
  55. 5
      js/map.js
  56. 1
      js/message.js
  57. 113
      objectivec/GPBMessage.m
  58. 9
      objectivec/GPBUtilities.m
  59. 4
      objectivec/GPBUtilities_PackagePrivate.h
  60. 7
      python/google/protobuf/internal/api_implementation.py
  61. 3
      python/google/protobuf/internal/containers.py
  62. 5
      python/google/protobuf/internal/message_test.py
  63. 5
      python/google/protobuf/pyext/map_container.cc
  64. 2
      python/google/protobuf/pyext/message.cc
  65. 5
      python/google/protobuf/text_format.py
  66. 3
      src/Makefile.am
  67. 5
      src/google/protobuf/any.cc
  68. 3
      src/google/protobuf/any.h
  69. 8
      src/google/protobuf/any.pb.cc
  70. 27
      src/google/protobuf/any.pb.h
  71. 7
      src/google/protobuf/any_lite.cc
  72. 8
      src/google/protobuf/any_test.cc
  73. 28
      src/google/protobuf/api.pb.cc
  74. 92
      src/google/protobuf/api.pb.h
  75. 298
      src/google/protobuf/arena.cc
  76. 144
      src/google/protobuf/arena.h
  77. 419
      src/google/protobuf/arena_impl.h
  78. 176
      src/google/protobuf/arena_unittest.cc
  79. 254
      src/google/protobuf/arenastring.cc
  80. 477
      src/google/protobuf/arenastring.h
  81. 119
      src/google/protobuf/arenastring_unittest.cc
  82. 26
      src/google/protobuf/compiler/command_line_interface.cc
  83. 3
      src/google/protobuf/compiler/command_line_interface.h
  84. 39
      src/google/protobuf/compiler/command_line_interface_unittest.cc
  85. 9
      src/google/protobuf/compiler/cpp/cpp_field.h
  86. 3
      src/google/protobuf/compiler/cpp/cpp_file.cc
  87. 22
      src/google/protobuf/compiler/cpp/cpp_helpers.cc
  88. 10
      src/google/protobuf/compiler/cpp/cpp_helpers.h
  89. 24
      src/google/protobuf/compiler/cpp/cpp_message.cc
  90. 3
      src/google/protobuf/compiler/cpp/cpp_message.h
  91. 205
      src/google/protobuf/compiler/cpp/cpp_string_field.cc
  92. 10
      src/google/protobuf/compiler/cpp/cpp_string_field.h
  93. 5
      src/google/protobuf/compiler/java/java_enum.cc
  94. 4
      src/google/protobuf/compiler/java/java_enum_field.cc
  95. 4
      src/google/protobuf/compiler/java/java_enum_field_lite.cc
  96. 5
      src/google/protobuf/compiler/java/java_enum_lite.cc
  97. 6
      src/google/protobuf/compiler/java/java_file.cc
  98. 1
      src/google/protobuf/compiler/java/java_file.h
  99. 1
      src/google/protobuf/compiler/java/java_message.cc
  100. 70
      src/google/protobuf/compiler/java/java_name_resolver.cc
  101. Some files were not shown because too many files have changed in this diff Show More

@ -164,6 +164,7 @@ cc_library(
# AUTOGEN(protobuf_lite_srcs) # AUTOGEN(protobuf_lite_srcs)
"src/google/protobuf/any_lite.cc", "src/google/protobuf/any_lite.cc",
"src/google/protobuf/arena.cc", "src/google/protobuf/arena.cc",
"src/google/protobuf/arenastring.cc",
"src/google/protobuf/extension_set.cc", "src/google/protobuf/extension_set.cc",
"src/google/protobuf/generated_enum_util.cc", "src/google/protobuf/generated_enum_util.cc",
"src/google/protobuf/generated_message_table_driven_lite.cc", "src/google/protobuf/generated_message_table_driven_lite.cc",

@ -3,10 +3,13 @@ Unreleased Changes
Protocol Compiler Protocol Compiler
* The proto compiler no longer requires a .proto filename when it is not * The proto compiler no longer requires a .proto filename when it is not
generating code. generating code.
* Added flag `--deterministic_output` to `protoc --encode=...`.
* Fixed deadlock when using google.protobuf.Any embedded in aggregate options.
C++ C++
* Arenas are now unconditionally enabled. cc_enable_arenas no longer has * Arenas are now unconditionally enabled. cc_enable_arenas no longer has
any effect. any effect.
* Removed inlined string support, which is incompatible with arenas.
* Fix a memory corruption bug in reflection when mixing optional and * Fix a memory corruption bug in reflection when mixing optional and
non-optional fields. non-optional fields.
* Make SpaceUsed() calculation more thorough for map fields. * Make SpaceUsed() calculation more thorough for map fields.
@ -20,9 +23,23 @@ Unreleased Changes
* When running under ASAN, skip a test that makes huge allocations. * When running under ASAN, skip a test that makes huge allocations.
* Fixed a crash that could happen when creating more than 256 extensions in * Fixed a crash that could happen when creating more than 256 extensions in
a single message. a single message.
* Fix a crash in BuildFile when passing in invalid descriptor proto.
* Parser security fix when operating with CodedInputStream.
* Warn against the use of AllowUnknownExtension.
* Migrated to C++11 for-range loops instead of index-based loops where
possible. This fixes a lot of warnings when compiling with -Wsign-compare.
Java Java
* Bugfix in mergeFrom() when a oneof has multiple message fields. * Bugfix in mergeFrom() when a oneof has multiple message fields.
* Fix RopeByteString.RopeInputStream.read() returning -1 when told to read
0 bytes when not at EOF.
* Redefine remove(Object) on primitive repeated field Lists to avoid
autoboxing.
* Support "\u" escapes in textformat string literals.
* Trailing empty spaces are no longer ignored for FieldMask.
* Fix FieldMaskUtil.subtract to recursively remove mask.
* Mark enums with `@java.lang.Deprecated` if the proto enum has option
`deprecated = true;`.
Python Python
* Print google.protobuf.NullValue as null instead of "NULL_VALUE" when it is * Print google.protobuf.NullValue as null instead of "NULL_VALUE" when it is

@ -53,7 +53,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tab
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\inlined_string_field.h" include\google\protobuf\inlined_string_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h
@ -88,7 +87,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" i
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\fastmem.h" include\google\protobuf\stubs\fastmem.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h

@ -1,6 +1,7 @@
set(libprotobuf_lite_files set(libprotobuf_lite_files
${protobuf_source_dir}/src/google/protobuf/any_lite.cc ${protobuf_source_dir}/src/google/protobuf/any_lite.cc
${protobuf_source_dir}/src/google/protobuf/arena.cc ${protobuf_source_dir}/src/google/protobuf/arena.cc
${protobuf_source_dir}/src/google/protobuf/arenastring.cc
${protobuf_source_dir}/src/google/protobuf/extension_set.cc ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc

@ -97,6 +97,10 @@ function(_protobuf_find_libraries name filename)
else() else()
get_target_property(${name}_LIBRARY_RELEASE protobuf::lib${filename} get_target_property(${name}_LIBRARY_RELEASE protobuf::lib${filename}
LOCATION_RELEASE) LOCATION_RELEASE)
get_target_property(${name}_LIBRARY_RELWITHDEBINFO protobuf::lib${filename}
LOCATION_RELWITHDEBINFO)
get_target_property(${name}_LIBRARY_MINSIZEREL protobuf::lib${filename}
LOCATION_MINSIZEREL)
get_target_property(${name}_LIBRARY_DEBUG protobuf::lib${filename} get_target_property(${name}_LIBRARY_DEBUG protobuf::lib${filename}
LOCATION_DEBUG) LOCATION_DEBUG)
@ -146,6 +150,14 @@ get_target_property(Protobuf_INCLUDE_DIRS protobuf::libprotobuf
# Set the protoc Executable # Set the protoc Executable
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
IMPORTED_LOCATION_RELEASE) IMPORTED_LOCATION_RELEASE)
if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
IMPORTED_LOCATION_RELWITHDEBINFO)
endif()
if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
IMPORTED_LOCATION_MINSIZEREL)
endif()
if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
IMPORTED_LOCATION_DEBUG) IMPORTED_LOCATION_DEBUG)

@ -333,7 +333,7 @@ conformance-php-c:
# Targets for actually running tests. # Targets for actually running tests.
test_cpp: protoc_middleman conformance-test-runner conformance-cpp test_cpp: protoc_middleman conformance-test-runner conformance-cpp
./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt ./conformance-cpp ./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt --text_format_failure_list text_format_failure_list_cpp.txt ./conformance-cpp
test_java: protoc_middleman conformance-test-runner conformance-java test_java: protoc_middleman conformance-test-runner conformance-java
./conformance-test-runner --enforce_recommended --failure_list failure_list_java.txt --text_format_failure_list text_format_failure_list_java.txt ./conformance-java ./conformance-test-runner --enforce_recommended --failure_list failure_list_java.txt --text_format_failure_list text_format_failure_list_java.txt ./conformance-java
@ -359,10 +359,24 @@ test_php_c_32: protoc_middleman conformance-test-runner conformance-php-c $(othe
# These depend on library paths being properly set up. The easiest way to # These depend on library paths being properly set up. The easiest way to
# run them is to just use "tox" from the python dir. # run them is to just use "tox" from the python dir.
test_python: protoc_middleman conformance-test-runner test_python: protoc_middleman conformance-test-runner
./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py VERSION="$(shell python --version 2>&1)"; \
if [[ "$$VERSION" == "Python 2.7"* ]]; then \
echo "Using Python 2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python_2.7.txt ./conformance_python.py; \
else \
echo "Using Python >2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py; \
fi
test_python_cpp: protoc_middleman conformance-test-runner test_python_cpp: protoc_middleman conformance-test-runner
./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt ./conformance_python.py VERSION="$(shell python --version 2>&1)"; \
if [[ "$$VERSION" == "Python 2.7"* ]]; then \
echo "Using Python 2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp_2.7.txt ./conformance_python.py; \
else \
echo "Using Python >2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp.txt ./conformance_python.py; \
fi
test_nodejs: protoc_middleman conformance-test-runner $(other_language_protoc_outputs) test_nodejs: protoc_middleman conformance-test-runner $(other_language_protoc_outputs)
NODE_PATH=../js:. ./conformance-test-runner --enforce_recommended --failure_list failure_list_js.txt ./conformance_nodejs.js NODE_PATH=../js:. ./conformance-test-runner --enforce_recommended --failure_list failure_list_js.txt ./conformance_nodejs.js

@ -258,6 +258,84 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() {
RunValidTextFormatTest("FloatFieldLargerThanUint64", REQUIRED, RunValidTextFormatTest("FloatFieldLargerThanUint64", REQUIRED,
"optional_float: 18446744073709551616"); "optional_float: 18446744073709551616");
// String literals x {Strings, Bytes}
for (const auto& field_type : std::vector<std::string>{"String", "Bytes"}) {
const std::string field_name =
field_type == "String" ? "optional_string" : "optional_bytes";
RunValidTextFormatTest(
StrCat("StringLiteralConcat", field_type), REQUIRED,
StrCat(field_name, ": 'first' \"second\"\n'third'"));
RunValidTextFormatTest(
StrCat("StringLiteralBasicEscapes", field_type), REQUIRED,
StrCat(field_name, ": '\\a\\b\\f\\n\\r\\t\\v\\?\\\\\\'\\\"'"));
RunValidTextFormatTest(
StrCat("StringLiteralOctalEscapes", field_type), REQUIRED,
StrCat(field_name, ": '\\341\\210\\264'"));
RunValidTextFormatTest(StrCat("StringLiteralHexEscapes", field_type),
REQUIRED,
StrCat(field_name, ": '\\xe1\\x88\\xb4'"));
RunValidTextFormatTest(
StrCat("StringLiteralShortUnicodeEscape", field_type),
RECOMMENDED, StrCat(field_name, ": '\\u1234'"));
RunValidTextFormatTest(
StrCat("StringLiteralLongUnicodeEscapes", field_type),
RECOMMENDED, StrCat(field_name, ": '\\U00001234\\U00010437'"));
// String literals don't include line feeds.
ExpectParseFailure(StrCat("StringLiteralIncludesLF", field_type),
REQUIRED,
StrCat(field_name, ": 'first line\nsecond line'"));
// Unicode escapes don't include code points that lie beyond the planes
// (> 0x10ffff).
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeTooLarge", field_type),
REQUIRED, StrCat(field_name, ": '\\U00110000'"));
// Unicode escapes don't include surrogates.
ExpectParseFailure(
StrCat("StringLiteralShortUnicodeEscapeSurrogatePair",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\ud801\\udc37'"));
ExpectParseFailure(
StrCat("StringLiteralShortUnicodeEscapeSurrogateFirstOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\ud800'"));
ExpectParseFailure(
StrCat("StringLiteralShortUnicodeEscapeSurrogateSecondOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\udc00'"));
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeSurrogateFirstOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000d800'"));
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeSurrogateSecondOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000dc00'"));
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeSurrogatePair", field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000d801\\U00000dc37'"));
ExpectParseFailure(
StrCat("StringLiteralUnicodeEscapeSurrogatePairLongShort",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000d801\\udc37'"));
ExpectParseFailure(
StrCat("StringLiteralUnicodeEscapeSurrogatePairShortLong",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\ud801\\U0000dc37'"));
// The following method depend on the type of field, as strings have extra
// validation.
const auto test_method =
field_type == "String"
? &TextFormatConformanceTestSuite::ExpectParseFailure
: &TextFormatConformanceTestSuite::RunValidTextFormatTest;
// String fields reject invalid UTF-8 byte sequences; bytes fields don't.
(this->*test_method)(StrCat(field_type, "FieldBadUTF8Octal"),
REQUIRED, StrCat(field_name, ": '\\300'"));
(this->*test_method)(StrCat(field_type, "FieldBadUTF8Hex"), REQUIRED,
StrCat(field_name, ": '\\xc0'"));
}
// Group fields // Group fields
RunValidTextFormatTestProto2("GroupFieldNoColon", REQUIRED, RunValidTextFormatTestProto2("GroupFieldNoColon", REQUIRED,
"Data { group_int32: 1 }"); "Data { group_int32: 1 }");

@ -0,0 +1,20 @@
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

@ -4,3 +4,10 @@ Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
Required.Proto3.TextFormatInput.AnyField.ProtobufOutput Required.Proto3.TextFormatInput.AnyField.ProtobufOutput
Required.Proto3.TextFormatInput.AnyField.TextFormatOutput Required.Proto3.TextFormatInput.AnyField.TextFormatOutput
Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

@ -3,3 +3,32 @@
# TODO: These should be fixed. # TODO: These should be fixed.
Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

@ -0,0 +1,36 @@
# This is the list of text format conformance tests that are known to fail right
# now.
# TODO: These should be fixed.
Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

@ -0,0 +1,28 @@
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

@ -0,0 +1,30 @@
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

@ -110,6 +110,18 @@ namespace Google.Protobuf
Assert.AreEqual(10, bs[0]); Assert.AreEqual(10, bs[0]);
} }
[Test]
public void CopyFromReadOnlySpanCopiesContents()
{
byte[] data = new byte[1];
data[0] = 10;
ReadOnlySpan<byte> byteSpan = data;
var bs = ByteString.CopyFrom(byteSpan);
Assert.AreEqual(10, bs[0]);
data[0] = 5;
Assert.AreEqual(10, bs[0]);
}
[Test] [Test]
public void ToByteArrayCopiesContents() public void ToByteArrayCopiesContents()
{ {

@ -245,6 +245,7 @@ namespace Google.Protobuf
/// are copied, so further modifications to the span will not /// are copied, so further modifications to the span will not
/// be reflected in the returned <see cref="ByteString" />. /// be reflected in the returned <see cref="ByteString" />.
/// </summary> /// </summary>
[SecuritySafeCritical]
public static ByteString CopyFrom(ReadOnlySpan<byte> bytes) public static ByteString CopyFrom(ReadOnlySpan<byte> bytes)
{ {
return new ByteString(bytes.ToArray()); return new ByteString(bytes.ToArray());

@ -69,10 +69,10 @@ syn keyword pbBool true false
syn match pbInt /-\?\<\d\+\>/ syn match pbInt /-\?\<\d\+\>/
syn match pbInt /\<0[xX]\x+\>/ syn match pbInt /\<0[xX]\x+\>/
syn match pbFloat /\<-\?\d*\(\.\d*\)\?/ syn match pbFloat /\<-\?\d*\(\.\d*\)\?/
syn region pbComment start="\/\*" end="\*\/" contains=@pbCommentGrp syn region pbComment start="\/\*" end="\*\/" contains=@pbCommentGrp,@Spell
syn region pbComment start="//" skip="\\$" end="$" keepend contains=@pbCommentGrp syn region pbComment start="//" skip="\\$" end="$" keepend contains=@pbCommentGrp,@Spell
syn region pbString start=/"/ skip=/\\./ end=/"/ syn region pbString start=/"/ skip=/\\./ end=/"/ contains=@Spell
syn region pbString start=/'/ skip=/\\./ end=/'/ syn region pbString start=/'/ skip=/\\./ end=/'/ contains=@Spell
if version >= 508 || !exists("did_proto_syn_inits") if version >= 508 || !exists("did_proto_syn_inits")
if version < 508 if version < 508

@ -142,7 +142,12 @@ abstract class AbstractProtobufList<E> extends AbstractList<E> implements Protob
@Override @Override
public boolean remove(Object o) { public boolean remove(Object o) {
ensureIsMutable(); ensureIsMutable();
return super.remove(o); int index = indexOf(o);
if (index == -1) {
return false;
}
remove(index);
return true;
} }
@Override @Override

@ -267,20 +267,6 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
return true; return true;
} }
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override @Override
public Boolean remove(int index) { public Boolean remove(int index) {
ensureIsMutable(); ensureIsMutable();

@ -41,6 +41,7 @@ import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -2009,14 +2010,14 @@ public abstract class CodedInputStream {
int prevPos = buffer.position(); int prevPos = buffer.position();
int prevLimit = buffer.limit(); int prevLimit = buffer.limit();
try { try {
buffer.position(bufferPos(begin)); ((Buffer) buffer).position(bufferPos(begin));
buffer.limit(bufferPos(end)); ((Buffer) buffer).limit(bufferPos(end));
return buffer.slice(); return buffer.slice();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
throw InvalidProtocolBufferException.truncatedMessage(); throw InvalidProtocolBufferException.truncatedMessage();
} finally { } finally {
buffer.position(prevPos); ((Buffer) buffer).position(prevPos);
buffer.limit(prevLimit); ((Buffer) buffer).limit(prevLimit);
} }
} }
} }
@ -3910,14 +3911,14 @@ public abstract class CodedInputStream {
int prevPos = currentByteBuffer.position(); int prevPos = currentByteBuffer.position();
int prevLimit = currentByteBuffer.limit(); int prevLimit = currentByteBuffer.limit();
try { try {
currentByteBuffer.position(begin); ((Buffer) currentByteBuffer).position(begin);
currentByteBuffer.limit(end); ((Buffer) currentByteBuffer).limit(end);
return currentByteBuffer.slice(); return currentByteBuffer.slice();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
throw InvalidProtocolBufferException.truncatedMessage(); throw InvalidProtocolBufferException.truncatedMessage();
} finally { } finally {
currentByteBuffer.position(prevPos); ((Buffer) currentByteBuffer).position(prevPos);
currentByteBuffer.limit(prevLimit); ((Buffer) currentByteBuffer).limit(prevLimit);
} }
} }
} }

@ -764,7 +764,10 @@ public final class Descriptors {
/** /**
* Finds a field by name. * Finds a field by name.
* *
* @param name The unqualified name of the field (e.g. "foo"). * @param name The unqualified name of the field (e.g. "foo"). For protocol buffer messages that
* follow <a
* href=https://developers.google.com/protocol-buffers/docs/style#message_and_field_names>Google's
* guidance on naming</a> this will be a snake case string, such as <pre>song_name</pre>.
* @return The field's descriptor, or {@code null} if not found. * @return The field's descriptor, or {@code null} if not found.
*/ */
public FieldDescriptor findFieldByName(final String name) { public FieldDescriptor findFieldByName(final String name) {

@ -267,20 +267,6 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
return true; return true;
} }
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override @Override
public Double remove(int index) { public Double remove(int index) {
ensureIsMutable(); ensureIsMutable();

@ -46,7 +46,6 @@ final class ExtensionRegistryFactory {
@Nullable */ @Nullable */
static final Class<?> EXTENSION_REGISTRY_CLASS = reflectExtensionRegistry(); static final Class<?> EXTENSION_REGISTRY_CLASS = reflectExtensionRegistry();
/* @Nullable */
static Class<?> reflectExtensionRegistry() { static Class<?> reflectExtensionRegistry() {
try { try {
return Class.forName(FULL_REGISTRY_CLASS_NAME); return Class.forName(FULL_REGISTRY_CLASS_NAME);
@ -77,7 +76,6 @@ final class ExtensionRegistryFactory {
&& EXTENSION_REGISTRY_CLASS.isAssignableFrom(registry.getClass()); && EXTENSION_REGISTRY_CLASS.isAssignableFrom(registry.getClass());
} }
/* @Nullable */
private static final ExtensionRegistryLite invokeSubclassFactory(String methodName) { private static final ExtensionRegistryLite invokeSubclassFactory(String methodName) {
if (EXTENSION_REGISTRY_CLASS == null) { if (EXTENSION_REGISTRY_CLASS == null) {
return null; return null;

@ -84,10 +84,8 @@ public class ExtensionRegistryLite {
static final String EXTENSION_CLASS_NAME = "com.google.protobuf.Extension"; static final String EXTENSION_CLASS_NAME = "com.google.protobuf.Extension";
private static class ExtensionClassHolder { private static class ExtensionClassHolder {
/* @Nullable */
static final Class<?> INSTANCE = resolveExtensionClass(); static final Class<?> INSTANCE = resolveExtensionClass();
/* @Nullable */
static Class<?> resolveExtensionClass() { static Class<?> resolveExtensionClass() {
try { try {
return Class.forName(EXTENSION_CLASS_NAME); return Class.forName(EXTENSION_CLASS_NAME);

@ -204,7 +204,6 @@ public enum FieldType {
* *
* @return the {@link FieldType} or {@code null} if not found. * @return the {@link FieldType} or {@code null} if not found.
*/ */
/* @Nullable */
public static FieldType forId(int id) { public static FieldType forId(int id) {
if (id < 0 || id >= VALUES.length) { if (id < 0 || id >= VALUES.length) {
return null; return null;
@ -228,7 +227,6 @@ public enum FieldType {
* *
* @return the generic super class/interface, or {@code null} if not found. * @return the generic super class/interface, or {@code null} if not found.
*/ */
/* @Nullable */
private static Type getGenericSuperList(Class<?> clazz) { private static Type getGenericSuperList(Class<?> clazz) {
// First look at interfaces. // First look at interfaces.
Type[] genericInterfaces = clazz.getGenericInterfaces(); Type[] genericInterfaces = clazz.getGenericInterfaces();

@ -266,20 +266,6 @@ final class FloatArrayList extends AbstractProtobufList<Float>
return true; return true;
} }
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override @Override
public Float remove(int index) { public Float remove(int index) {
ensureIsMutable(); ensureIsMutable();

@ -266,20 +266,6 @@ final class IntArrayList extends AbstractProtobufList<Integer>
return true; return true;
} }
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override @Override
public Integer remove(int index) { public Integer remove(int index) {
ensureIsMutable(); ensureIsMutable();

@ -30,7 +30,6 @@
package com.google.protobuf; package com.google.protobuf;
import java.io.IOException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.Charset; import java.nio.charset.Charset;

@ -266,20 +266,6 @@ final class LongArrayList extends AbstractProtobufList<Long>
return true; return true;
} }
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override @Override
public Long remove(int index) { public Long remove(int index) {
ensureIsMutable(); ensureIsMutable();

@ -2572,7 +2572,7 @@ final class MessageSchema<T> implements Schema<T> {
int presenceMaskAndOffset = 0; int presenceMaskAndOffset = 0;
int presenceMask = 0; int presenceMask = 0;
if (!proto3 && fieldType <= 17) { if (fieldType <= 17) {
presenceMaskAndOffset = buffer[pos + 2]; presenceMaskAndOffset = buffer[pos + 2];
final int presenceFieldOffset = presenceMaskAndOffset & OFFSET_MASK; final int presenceFieldOffset = presenceMaskAndOffset & OFFSET_MASK;
if (presenceFieldOffset != currentPresenceFieldOffset) { if (presenceFieldOffset != currentPresenceFieldOffset) {

@ -37,6 +37,7 @@ import java.io.InputStream;
import java.io.InvalidObjectException; import java.io.InvalidObjectException;
import java.io.ObjectInputStream; import java.io.ObjectInputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
import java.nio.InvalidMarkException; import java.nio.InvalidMarkException;
@ -109,7 +110,7 @@ final class NioByteString extends ByteString.LeafByteString {
protected void copyToInternal( protected void copyToInternal(
byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { byte[] target, int sourceOffset, int targetOffset, int numberToCopy) {
ByteBuffer slice = buffer.slice(); ByteBuffer slice = buffer.slice();
slice.position(sourceOffset); ((Buffer) slice).position(sourceOffset);
slice.get(target, targetOffset, numberToCopy); slice.get(target, targetOffset, numberToCopy);
} }
@ -285,8 +286,8 @@ final class NioByteString extends ByteString.LeafByteString {
} }
ByteBuffer slice = buffer.slice(); ByteBuffer slice = buffer.slice();
slice.position(beginIndex - buffer.position()); ((Buffer) slice).position(beginIndex - buffer.position());
slice.limit(endIndex - buffer.position()); ((Buffer) slice).limit(endIndex - buffer.position());
return slice; return slice;
} }
} }

@ -76,11 +76,8 @@ final class Protobuf {
schemaFor(message).makeImmutable(message); schemaFor(message).makeImmutable(message);
} }
/** /** Checks if all required fields are set. */
* Checks if all required fields are set. TODO(xiaofeng): Make this package private when the tests <T> boolean isInitialized(T message) {
* are moved to protobuf package.
*/
public <T> boolean isInitialized(T message) {
return schemaFor(message).isInitialized(message); return schemaFor(message).isInitialized(message);
} }

@ -845,7 +845,10 @@ final class RopeByteString extends ByteString {
throw new IndexOutOfBoundsException(); throw new IndexOutOfBoundsException();
} }
int bytesRead = readSkipInternal(b, offset, length); int bytesRead = readSkipInternal(b, offset, length);
if (bytesRead == 0) { if (bytesRead == 0 && (length > 0 || availableInternal() == 0)) {
// Modeling ByteArrayInputStream.read(byte[], int, int) behavior noted above:
// It's ok to read 0 bytes on purpose (length == 0) from a stream that isn't at EOF.
// It's not ok to try to read bytes (even 0 bytes) from a stream that is at EOF.
return -1; return -1;
} else { } else {
return bytesRead; return bytesRead;
@ -905,8 +908,7 @@ final class RopeByteString extends ByteString {
@Override @Override
public int available() throws IOException { public int available() throws IOException {
int bytesRead = currentPieceOffsetInRope + currentPieceIndex; return availableInternal();
return RopeByteString.this.size() - bytesRead;
} }
@Override @Override
@ -955,5 +957,11 @@ final class RopeByteString extends ByteString {
} }
} }
} }
/** Computes the number of bytes still available to read. */
private int availableInternal() {
int bytesRead = currentPieceOffsetInRope + currentPieceIndex;
return RopeByteString.this.size() - bytesRead;
}
} }
} }

@ -30,6 +30,8 @@
package com.google.protobuf; package com.google.protobuf;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumDescriptor;
import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor;
@ -2375,6 +2377,73 @@ public final class TextFormat {
result[pos++] = (byte) code; result[pos++] = (byte) code;
break; break;
case 'u':
// Unicode escape
++i;
if (i + 3 < input.size()
&& isHex(input.byteAt(i))
&& isHex(input.byteAt(i + 1))
&& isHex(input.byteAt(i + 2))
&& isHex(input.byteAt(i + 3))) {
char ch =
(char)
(digitValue(input.byteAt(i)) << 12
| digitValue(input.byteAt(i + 1)) << 8
| digitValue(input.byteAt(i + 2)) << 4
| digitValue(input.byteAt(i + 3)));
if (Character.isSurrogate(ch)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\u' refers to a surrogate");
}
byte[] chUtf8 = Character.toString(ch).getBytes(UTF_8);
System.arraycopy(chUtf8, 0, result, pos, chUtf8.length);
pos += chUtf8.length;
i += 3;
} else {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\u' with too few hex chars");
}
break;
case 'U':
// Unicode escape
++i;
if (i + 7 >= input.size()) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U' with too few hex chars");
}
int codepoint = 0;
for (int offset = i; offset < i + 8; offset++) {
byte b = input.byteAt(offset);
if (!isHex(b)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U' with too few hex chars");
}
codepoint = (codepoint << 4) | digitValue(b);
}
if (!Character.isValidCodePoint(codepoint)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U"
+ input.substring(i, i + 8).toStringUtf8()
+ "' is not a valid code point value");
}
Character.UnicodeBlock unicodeBlock = Character.UnicodeBlock.of(codepoint);
if (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES)
|| unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES)
|| unicodeBlock.equals(Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U"
+ input.substring(i, i + 8).toStringUtf8()
+ "' refers to a surrogate code unit");
}
int[] codepoints = new int[1];
codepoints[0] = codepoint;
byte[] chUtf8 = new String(codepoints, 0, 1).getBytes(UTF_8);
System.arraycopy(chUtf8, 0, result, pos, chUtf8.length);
pos += chUtf8.length;
i += 7;
break;
default: default:
throw new InvalidEscapeSequenceException( throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\" + (char) c + '\''); "Invalid escape sequence: '\\" + (char) c + '\'');

@ -70,7 +70,6 @@ public class TypeRegistry {
/** /**
* Find a type by its typeUrl. Returns null if it cannot be found in this {@link TypeRegistry}. * Find a type by its typeUrl. Returns null if it cannot be found in this {@link TypeRegistry}.
*/ */
/* @Nullable */
public final Descriptor getDescriptorForTypeUrl(String typeUrl) public final Descriptor getDescriptorForTypeUrl(String typeUrl)
throws InvalidProtocolBufferException { throws InvalidProtocolBufferException {
return find(getTypeName(typeUrl)); return find(getTypeName(typeUrl));

@ -41,7 +41,6 @@ import java.util.logging.Logger;
/** Utility class for working with unsafe operations. */ /** Utility class for working with unsafe operations. */
final class UnsafeUtil { final class UnsafeUtil {
private static final Logger logger = Logger.getLogger(UnsafeUtil.class.getName());
private static final sun.misc.Unsafe UNSAFE = getUnsafe(); private static final sun.misc.Unsafe UNSAFE = getUnsafe();
private static final Class<?> MEMORY_CLASS = Android.getMemoryClass(); private static final Class<?> MEMORY_CLASS = Android.getMemoryClass();
private static final boolean IS_ANDROID_64 = determineAndroidSupportByAddressSize(long.class); private static final boolean IS_ANDROID_64 = determineAndroidSupportByAddressSize(long.class);
@ -363,7 +362,11 @@ final class UnsafeUtil {
return true; return true;
} catch (Throwable e) { } catch (Throwable e) {
logger.log( // Because log statements are fairly sparse in this class, this logger is initialized
// non-statically. Static initialization adds undue runtime costs to the first client to
// initialize this class.
Logger.getLogger(UnsafeUtil.class.getName())
.log(
Level.WARNING, Level.WARNING,
"platform method missing - proto runtime falling back to safer methods: " + e); "platform method missing - proto runtime falling back to safer methods: " + e);
} }
@ -397,7 +400,11 @@ final class UnsafeUtil {
clazz.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class); clazz.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class);
return true; return true;
} catch (Throwable e) { } catch (Throwable e) {
logger.log( // Because log statements are fairly sparse in this class, this logger is initialized
// non-statically. Static initialization adds undue runtime costs to the first client to
// initialize this class.
Logger.getLogger(UnsafeUtil.class.getName())
.log(
Level.WARNING, Level.WARNING,
"platform method missing - proto runtime falling back to safer methods: " + e); "platform method missing - proto runtime falling back to safer methods: " + e);
} }

@ -514,6 +514,20 @@ public class LiteralByteStringTest extends TestCase {
assertEquals(classUnderTest + " InputStream must now be exhausted", -1, input.read()); assertEquals(classUnderTest + " InputStream must now be exhausted", -1, input.read());
} }
public void testNewInput_readZeroBytes() throws IOException {
InputStream input = stringUnderTest.newInput();
assertEquals(
classUnderTest + " InputStream.read() returns 0 when told to read 0 bytes and not at EOF",
0,
input.read(new byte[0]));
input.skip(input.available());
assertEquals(
classUnderTest + " InputStream.read() returns -1 when told to read 0 bytes at EOF",
-1,
input.read(new byte[0]));
}
public void testNewInput_skip() throws IOException { public void testNewInput_skip() throws IOException {
InputStream input = stringUnderTest.newInput(); InputStream input = stringUnderTest.newInput();
int stringSize = stringUnderTest.size(); int stringSize = stringUnderTest.size();

@ -77,44 +77,44 @@ public final class MapForProto2LiteTest extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) { private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field()) .putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringField()) .putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesField()) .putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumField()) .putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageField()) .putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32Field()); .putAllStringToInt32Field(source.getStringToInt32FieldMap());
} }
private void assertMapValuesSet(TestMap message) { private void assertMapValuesSet(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(11, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(11, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(22, message.getInt32ToInt32Field().get(2).intValue()); assertEquals(22, message.getInt32ToInt32FieldMap().get(2).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("11", message.getInt32ToStringField().get(1)); assertEquals("11", message.getInt32ToStringFieldMap().get(1));
assertEquals("22", message.getInt32ToStringField().get(2)); assertEquals("22", message.getInt32ToStringFieldMap().get(2));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2)); assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesFieldMap().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(11, message.getInt32ToMessageField().get(1).getValue()); assertEquals(11, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(22, message.getInt32ToMessageField().get(2).getValue()); assertEquals(22, message.getInt32ToMessageFieldMap().get(2).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(11, message.getStringToInt32Field().get("1").intValue()); assertEquals(11, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(22, message.getStringToInt32Field().get("2").intValue()); assertEquals(22, message.getStringToInt32FieldMap().get("2").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
} }
private void updateMapValues(TestMap.Builder builder) { private void updateMapValues(TestMap.Builder builder) {
@ -152,35 +152,35 @@ public final class MapForProto2LiteTest extends TestCase {
} }
private void assertMapValuesUpdated(TestMap message) { private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue()); assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringField().get(1)); assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringField().get(4)); assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4)); assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4)); assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue()); assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue()); assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue()); assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue()); assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
} }
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) { private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -206,13 +206,13 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap.Builder builder = TestMap.newBuilder(); TestMap.Builder builder = TestMap.newBuilder();
TestMap message = builder.build(); TestMap message = builder.build();
builder.putInt32ToInt32Field(1, 2); builder.putInt32ToInt32Field(1, 2);
assertTrue(message.getInt32ToInt32Field().isEmpty()); assertTrue(message.getInt32ToInt32FieldMap().isEmpty());
message = builder.build(); message = builder.build();
assertEquals(newMap(1, 2), message.getInt32ToInt32Field()); assertEquals(newMap(1, 2), message.getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
builder.putInt32ToInt32Field(2, 3); builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2), message.getInt32ToInt32Field()); assertEquals(newMap(1, 2), message.getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
} }
public void testGetMapIsImmutable() { public void testGetMapIsImmutable() {
@ -254,30 +254,31 @@ public final class MapForProto2LiteTest extends TestCase {
public void testMutableMapLifecycle() { public void testMutableMapLifecycle() {
TestMap.Builder builder = TestMap.newBuilder().putInt32ToInt32Field(1, 2); TestMap.Builder builder = TestMap.newBuilder().putInt32ToInt32Field(1, 2);
assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field()); assertEquals(newMap(1, 2), builder.build().getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
builder.putInt32ToInt32Field(2, 3); builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
builder.putInt32ToEnumField(1, TestMap.EnumValue.BAR); builder.putInt32ToEnumField(1, TestMap.EnumValue.BAR);
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField()); assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumFieldMap());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField()); assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumFieldMap());
builder.putInt32ToEnumField(2, TestMap.EnumValue.FOO); builder.putInt32ToEnumField(2, TestMap.EnumValue.FOO);
assertEquals( assertEquals(
newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO), builder.getInt32ToEnumField()); newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
builder.getInt32ToEnumFieldMap());
builder.putInt32ToStringField(1, "1"); builder.putInt32ToStringField(1, "1");
assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField()); assertEquals(newMap(1, "1"), builder.build().getInt32ToStringFieldMap());
assertEquals(newMap(1, "1"), builder.getInt32ToStringField()); assertEquals(newMap(1, "1"), builder.getInt32ToStringFieldMap());
builder.putInt32ToStringField(2, "2"); builder.putInt32ToStringField(2, "2");
assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringField()); assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringFieldMap());
builder.putInt32ToMessageField(1, TestMap.MessageValue.getDefaultInstance()); builder.putInt32ToMessageField(1, TestMap.MessageValue.getDefaultInstance());
assertEquals( assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()), newMap(1, TestMap.MessageValue.getDefaultInstance()),
builder.build().getInt32ToMessageField()); builder.build().getInt32ToMessageFieldMap());
assertEquals( assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageField()); newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageFieldMap());
builder.putInt32ToMessageField(2, TestMap.MessageValue.getDefaultInstance()); builder.putInt32ToMessageField(2, TestMap.MessageValue.getDefaultInstance());
assertEquals( assertEquals(
newMap( newMap(
@ -285,7 +286,7 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap.MessageValue.getDefaultInstance(), TestMap.MessageValue.getDefaultInstance(),
2, 2,
TestMap.MessageValue.getDefaultInstance()), TestMap.MessageValue.getDefaultInstance()),
builder.getInt32ToMessageField()); builder.getInt32ToMessageFieldMap());
} }
public void testGettersAndSetters() throws Exception { public void testGettersAndSetters() throws Exception {
@ -415,7 +416,7 @@ public final class MapForProto2LiteTest extends TestCase {
} catch (InvalidProtocolBufferException expected) { } catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap); assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage(); map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty()); assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
} }
map = map =
@ -476,14 +477,14 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap message = TestMap.parseFrom(data); TestMap message = TestMap.parseFrom(data);
// Entries with unknown enum values will be stored into UnknownFieldSet so // Entries with unknown enum values will be stored into UnknownFieldSet so
// there is only one entry in the map. // there is only one entry in the map.
assertEquals(1, message.getInt32ToEnumField().size()); assertEquals(1, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
// Serializing and parsing should preserve the unknown entry. // Serializing and parsing should preserve the unknown entry.
data = message.toByteString(); data = message.toByteString();
TestUnknownEnumValue messageWithUnknownEnums = TestUnknownEnumValue.parseFrom(data); TestUnknownEnumValue messageWithUnknownEnums = TestUnknownEnumValue.parseFrom(data);
assertEquals(2, messageWithUnknownEnums.getInt32ToInt32Field().size()); assertEquals(2, messageWithUnknownEnums.getInt32ToInt32FieldMap().size());
assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue()); assertEquals(1, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue()); assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(2).intValue());
} }
public void testIterationOrder() throws Exception { public void testIterationOrder() throws Exception {
@ -493,7 +494,7 @@ public final class MapForProto2LiteTest extends TestCase {
assertEquals( assertEquals(
Arrays.asList("1", "2", "3"), Arrays.asList("1", "2", "3"),
new ArrayList<String>(message.getStringToInt32Field().keySet())); new ArrayList<String>(message.getStringToInt32FieldMap().keySet()));
} }
private static <K, V> Map<K, V> newMap(K key1, V value1) { private static <K, V> Map<K, V> newMap(K key1, V value1) {
@ -513,10 +514,10 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap.Builder builder = TestMap.newBuilder(); TestMap.Builder builder = TestMap.newBuilder();
setMapValues(builder); setMapValues(builder);
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap()); assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap()); assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap()); assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap()); assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
} }
public void testContains() { public void testContains() {

@ -119,12 +119,12 @@ public class MapForProto2Test extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) { private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field()) .putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringField()) .putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesField()) .putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumField()) .putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageField()) .putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32Field()); .putAllStringToInt32Field(source.getStringToInt32FieldMap());
} }
private void assertMapValuesSet(TestMapOrBuilder message) { private void assertMapValuesSet(TestMapOrBuilder message) {
@ -236,35 +236,35 @@ public class MapForProto2Test extends TestCase {
} }
private void assertMapValuesUpdated(TestMap message) { private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue()); assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringField().get(1)); assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringField().get(4)); assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4)); assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4)); assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue()); assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue()); assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue()); assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue()); assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
} }
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) { private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -563,7 +563,7 @@ public class MapForProto2Test extends TestCase {
} catch (InvalidProtocolBufferException expected) { } catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap); assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage(); map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty()); assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
} }
map = map =
@ -698,8 +698,8 @@ public class MapForProto2Test extends TestCase {
builder.clearField(f("int32_to_int32_field")); builder.clearField(f("int32_to_int32_field"));
builder.clearField(f("int32_to_message_field")); builder.clearField(f("int32_to_message_field"));
message = builder.build(); message = builder.build();
assertEquals(0, message.getInt32ToInt32Field().size()); assertEquals(0, message.getInt32ToInt32FieldMap().size());
assertEquals(0, message.getInt32ToMessageField().size()); assertEquals(0, message.getInt32ToMessageFieldMap().size());
// Test setField() // Test setField()
setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44)); setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44));
@ -710,10 +710,10 @@ public class MapForProto2Test extends TestCase {
111, MessageValue.newBuilder().setValue(222).build(), 111, MessageValue.newBuilder().setValue(222).build(),
333, MessageValue.newBuilder().setValue(444).build())); 333, MessageValue.newBuilder().setValue(444).build()));
message = builder.build(); message = builder.build();
assertEquals(22, message.getInt32ToInt32Field().get(11).intValue()); assertEquals(22, message.getInt32ToInt32FieldMap().get(11).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(33).intValue()); assertEquals(44, message.getInt32ToInt32FieldMap().get(33).intValue());
assertEquals(222, message.getInt32ToMessageField().get(111).getValue()); assertEquals(222, message.getInt32ToMessageFieldMap().get(111).getValue());
assertEquals(444, message.getInt32ToMessageField().get(333).getValue()); assertEquals(444, message.getInt32ToMessageFieldMap().get(333).getValue());
// Test addRepeatedField // Test addRepeatedField
builder.addRepeatedField( builder.addRepeatedField(
@ -726,8 +726,8 @@ public class MapForProto2Test extends TestCase {
555, 555,
MessageValue.newBuilder().setValue(666).build())); MessageValue.newBuilder().setValue(666).build()));
message = builder.build(); message = builder.build();
assertEquals(66, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(66, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(666, message.getInt32ToMessageField().get(555).getValue()); assertEquals(666, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test addRepeatedField (overriding existing values) // Test addRepeatedField (overriding existing values)
builder.addRepeatedField( builder.addRepeatedField(
@ -740,8 +740,8 @@ public class MapForProto2Test extends TestCase {
555, 555,
MessageValue.newBuilder().setValue(555).build())); MessageValue.newBuilder().setValue(555).build()));
message = builder.build(); message = builder.build();
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(555, message.getInt32ToMessageField().get(555).getValue()); assertEquals(555, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test setRepeatedField // Test setRepeatedField
for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) { for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) {
@ -755,9 +755,9 @@ public class MapForProto2Test extends TestCase {
builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build()); builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build());
} }
message = builder.build(); message = builder.build();
assertEquals(11, message.getInt32ToInt32Field().get(22).intValue()); assertEquals(11, message.getInt32ToInt32FieldMap().get(22).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(44).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(44).intValue());
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
} }
// See additional coverage in TextFormatTest.java. // See additional coverage in TextFormatTest.java.
@ -844,16 +844,16 @@ public class MapForProto2Test extends TestCase {
TestMap message = TestMap.parseFrom(data); TestMap message = TestMap.parseFrom(data);
// Entries with unknown enum values will be stored into UnknownFieldSet so // Entries with unknown enum values will be stored into UnknownFieldSet so
// there is only one entry in the map. // there is only one entry in the map.
assertEquals(1, message.getInt32ToEnumField().size()); assertEquals(1, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
// UnknownFieldSet should not be empty. // UnknownFieldSet should not be empty.
assertFalse(message.getUnknownFields().asMap().isEmpty()); assertFalse(message.getUnknownFields().asMap().isEmpty());
// Serializing and parsing should preserve the unknown entry. // Serializing and parsing should preserve the unknown entry.
data = message.toByteString(); data = message.toByteString();
TestUnknownEnumValue messageWithUnknownEnums = TestUnknownEnumValue.parseFrom(data); TestUnknownEnumValue messageWithUnknownEnums = TestUnknownEnumValue.parseFrom(data);
assertEquals(2, messageWithUnknownEnums.getInt32ToInt32Field().size()); assertEquals(2, messageWithUnknownEnums.getInt32ToInt32FieldMap().size());
assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue()); assertEquals(1, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue()); assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(2).intValue());
} }
public void testRequiredMessage() throws Exception { public void testRequiredMessage() throws Exception {
@ -885,7 +885,7 @@ public class MapForProto2Test extends TestCase {
assertEquals( assertEquals(
Arrays.asList("1", "2", "3"), Arrays.asList("1", "2", "3"),
new ArrayList<String>(message.getStringToInt32Field().keySet())); new ArrayList<String>(message.getStringToInt32FieldMap().keySet()));
} }
public void testContains() { public void testContains() {
@ -1172,9 +1172,9 @@ public class MapForProto2Test extends TestCase {
setMapValuesUsingAccessors(builder); setMapValuesUsingAccessors(builder);
assertMapValuesSet(builder); assertMapValuesSet(builder);
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap()); assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap()); assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap()); assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap()); assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
} }
} }

@ -84,44 +84,44 @@ public final class MapLiteTest extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) { private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field()) .putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringField()) .putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesField()) .putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumField()) .putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageField()) .putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32Field()); .putAllStringToInt32Field(source.getStringToInt32FieldMap());
} }
private void assertMapValuesSet(TestMap message) { private void assertMapValuesSet(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(11, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(11, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(22, message.getInt32ToInt32Field().get(2).intValue()); assertEquals(22, message.getInt32ToInt32FieldMap().get(2).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("11", message.getInt32ToStringField().get(1)); assertEquals("11", message.getInt32ToStringFieldMap().get(1));
assertEquals("22", message.getInt32ToStringField().get(2)); assertEquals("22", message.getInt32ToStringFieldMap().get(2));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2)); assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesFieldMap().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(11, message.getInt32ToMessageField().get(1).getValue()); assertEquals(11, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(22, message.getInt32ToMessageField().get(2).getValue()); assertEquals(22, message.getInt32ToMessageFieldMap().get(2).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(11, message.getStringToInt32Field().get("1").intValue()); assertEquals(11, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(22, message.getStringToInt32Field().get("2").intValue()); assertEquals(22, message.getStringToInt32FieldMap().get("2").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
} }
private void updateMapValues(TestMap.Builder builder) { private void updateMapValues(TestMap.Builder builder) {
@ -159,35 +159,35 @@ public final class MapLiteTest extends TestCase {
} }
private void assertMapValuesUpdated(TestMap message) { private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue()); assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringField().get(1)); assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringField().get(4)); assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4)); assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4)); assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue()); assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue()); assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue()); assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue()); assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
} }
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) { private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -213,12 +213,12 @@ public final class MapLiteTest extends TestCase {
TestMap.Builder builder = TestMap.newBuilder(); TestMap.Builder builder = TestMap.newBuilder();
TestMap message = builder.build(); TestMap message = builder.build();
builder.putInt32ToInt32Field(1, 2); builder.putInt32ToInt32Field(1, 2);
assertTrue(message.getInt32ToInt32Field().isEmpty()); assertTrue(message.getInt32ToInt32FieldMap().isEmpty());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
message = builder.build(); message = builder.build();
builder.putInt32ToInt32Field(2, 3); builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2), message.getInt32ToInt32Field()); assertEquals(newMap(1, 2), message.getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
} }
public void testGetMapIsImmutable() { public void testGetMapIsImmutable() {
@ -266,30 +266,31 @@ public final class MapLiteTest extends TestCase {
public void testMutableMapLifecycle() { public void testMutableMapLifecycle() {
TestMap.Builder builder = TestMap.newBuilder().putInt32ToInt32Field(1, 2); TestMap.Builder builder = TestMap.newBuilder().putInt32ToInt32Field(1, 2);
assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field()); assertEquals(newMap(1, 2), builder.build().getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
builder.putInt32ToInt32Field(2, 3); builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field()); assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
builder.putInt32ToEnumField(1, TestMap.EnumValue.BAR); builder.putInt32ToEnumField(1, TestMap.EnumValue.BAR);
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField()); assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumFieldMap());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField()); assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumFieldMap());
builder.putInt32ToEnumField(2, TestMap.EnumValue.FOO); builder.putInt32ToEnumField(2, TestMap.EnumValue.FOO);
assertEquals( assertEquals(
newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO), builder.getInt32ToEnumField()); newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
builder.getInt32ToEnumFieldMap());
builder.putInt32ToStringField(1, "1"); builder.putInt32ToStringField(1, "1");
assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField()); assertEquals(newMap(1, "1"), builder.build().getInt32ToStringFieldMap());
assertEquals(newMap(1, "1"), builder.getInt32ToStringField()); assertEquals(newMap(1, "1"), builder.getInt32ToStringFieldMap());
builder.putInt32ToStringField(2, "2"); builder.putInt32ToStringField(2, "2");
assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringField()); assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringFieldMap());
builder.putInt32ToMessageField(1, TestMap.MessageValue.getDefaultInstance()); builder.putInt32ToMessageField(1, TestMap.MessageValue.getDefaultInstance());
assertEquals( assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()), newMap(1, TestMap.MessageValue.getDefaultInstance()),
builder.build().getInt32ToMessageField()); builder.build().getInt32ToMessageFieldMap());
assertEquals( assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageField()); newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageFieldMap());
builder.putInt32ToMessageField(2, TestMap.MessageValue.getDefaultInstance()); builder.putInt32ToMessageField(2, TestMap.MessageValue.getDefaultInstance());
assertEquals( assertEquals(
newMap( newMap(
@ -297,7 +298,7 @@ public final class MapLiteTest extends TestCase {
TestMap.MessageValue.getDefaultInstance(), TestMap.MessageValue.getDefaultInstance(),
2, 2,
TestMap.MessageValue.getDefaultInstance()), TestMap.MessageValue.getDefaultInstance()),
builder.getInt32ToMessageField()); builder.getInt32ToMessageFieldMap());
} }
public void testGettersAndSetters() throws Exception { public void testGettersAndSetters() throws Exception {
@ -342,12 +343,12 @@ public final class MapLiteTest extends TestCase {
TestMap source = sourceBuilder.build(); TestMap source = sourceBuilder.build();
TestMap.Builder destinationBuilder = TestMap.newBuilder(); TestMap.Builder destinationBuilder = TestMap.newBuilder();
destinationBuilder.putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValue()); destinationBuilder.putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValueMap());
TestMap destination = destinationBuilder.build(); TestMap destination = destinationBuilder.build();
assertEquals(0, destination.getInt32ToEnumFieldValue().get(0).intValue()); assertEquals(0, destination.getInt32ToEnumFieldValueMap().get(0).intValue());
assertEquals(1, destination.getInt32ToEnumFieldValue().get(1).intValue()); assertEquals(1, destination.getInt32ToEnumFieldValueMap().get(1).intValue());
assertEquals(1000, destination.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, destination.getInt32ToEnumFieldValueMap().get(2).intValue());
assertEquals(3, destination.getInt32ToEnumFieldCount()); assertEquals(3, destination.getInt32ToEnumFieldCount());
} }
@ -459,7 +460,7 @@ public final class MapLiteTest extends TestCase {
} catch (InvalidProtocolBufferException expected) { } catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap); assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage(); map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty()); assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
} }
map = map =
@ -524,24 +525,24 @@ public final class MapLiteTest extends TestCase {
.putInt32ToEnumFieldValue(2, 1000); // unknown value. .putInt32ToEnumFieldValue(2, 1000); // unknown value.
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(0)); assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(0));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumFieldMap().get(2));
builder.putAllInt32ToEnumFieldValue(newMap(2, 1000)); // unknown value. builder.putAllInt32ToEnumFieldValue(newMap(2, 1000)); // unknown value.
message = builder.build(); message = builder.build();
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumFieldMap().get(2));
// Unknown enum values should be preserved after: // Unknown enum values should be preserved after:
// 1. Serialization and parsing. // 1. Serialization and parsing.
// 2. toBuild(). // 2. toBuild().
// 3. mergeFrom(). // 3. mergeFrom().
message = TestMap.parseFrom(message.toByteString()); message = TestMap.parseFrom(message.toByteString());
assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, message.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = message.toBuilder(); builder = message.toBuilder();
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = TestMap.newBuilder().mergeFrom(message); builder = TestMap.newBuilder().mergeFrom(message);
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
// hashCode()/equals() should take unknown enum values into account. // hashCode()/equals() should take unknown enum values into account.
builder.putAllInt32ToEnumFieldValue(newMap(2, 1001)); builder.putAllInt32ToEnumFieldValue(newMap(2, 1001));
@ -550,7 +551,7 @@ public final class MapLiteTest extends TestCase {
assertFalse(message.equals(message2)); assertFalse(message.equals(message2));
// Unknown values will be converted to UNRECOGNIZED so the resulted enum map // Unknown values will be converted to UNRECOGNIZED so the resulted enum map
// should be the same. // should be the same.
assertEquals(message2.getInt32ToEnumField(), message.getInt32ToEnumField()); assertEquals(message2.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
} }
public void testIterationOrder() throws Exception { public void testIterationOrder() throws Exception {
@ -559,18 +560,18 @@ public final class MapLiteTest extends TestCase {
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals( assertEquals(
Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32Field().keySet())); Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32FieldMap().keySet()));
} }
public void testGetMap() { public void testGetMap() {
TestMap.Builder builder = TestMap.newBuilder(); TestMap.Builder builder = TestMap.newBuilder();
setMapValues(builder); setMapValues(builder);
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap()); assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap()); assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap()); assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToEnumFieldValue(), message.getInt32ToEnumFieldValueMap()); assertEquals(message.getInt32ToEnumFieldValueMap(), message.getInt32ToEnumFieldValueMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap()); assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
} }
public void testContains() { public void testContains() {

@ -122,44 +122,44 @@ public class MapTest extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) { private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field()) .putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringField()) .putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesField()) .putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumField()) .putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageField()) .putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32Field()); .putAllStringToInt32Field(source.getStringToInt32FieldMap());
} }
private void assertMapValuesSet(TestMap message) { private void assertMapValuesSet(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(11, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(11, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(22, message.getInt32ToInt32Field().get(2).intValue()); assertEquals(22, message.getInt32ToInt32FieldMap().get(2).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("11", message.getInt32ToStringField().get(1)); assertEquals("11", message.getInt32ToStringFieldMap().get(1));
assertEquals("22", message.getInt32ToStringField().get(2)); assertEquals("22", message.getInt32ToStringFieldMap().get(2));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2)); assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesFieldMap().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(11, message.getInt32ToMessageField().get(1).getValue()); assertEquals(11, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(22, message.getInt32ToMessageField().get(2).getValue()); assertEquals(22, message.getInt32ToMessageFieldMap().get(2).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(11, message.getStringToInt32Field().get("1").intValue()); assertEquals(11, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(22, message.getStringToInt32Field().get("2").intValue()); assertEquals(22, message.getStringToInt32FieldMap().get("2").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
} }
private void updateMapValuesUsingMutableMap(TestMap.Builder builder) { private void updateMapValuesUsingMutableMap(TestMap.Builder builder) {
@ -239,35 +239,35 @@ public class MapTest extends TestCase {
} }
private void assertMapValuesUpdated(TestMap message) { private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size()); assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue()); assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue()); assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size()); assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringField().get(1)); assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringField().get(4)); assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4)); assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4)); assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue()); assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue()); assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size()); assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue()); assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue()); assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue()); assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
} }
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) { private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -468,11 +468,13 @@ public class MapTest extends TestCase {
.build(); .build();
TestMap destination = TestMap destination =
TestMap.newBuilder().putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValue()).build(); TestMap.newBuilder()
.putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValueMap())
.build();
assertEquals(0, destination.getInt32ToEnumFieldValue().get(0).intValue()); assertEquals(0, destination.getInt32ToEnumFieldValueMap().get(0).intValue());
assertEquals(1, destination.getInt32ToEnumFieldValue().get(1).intValue()); assertEquals(1, destination.getInt32ToEnumFieldValueMap().get(1).intValue());
assertEquals(1000, destination.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, destination.getInt32ToEnumFieldValueMap().get(2).intValue());
assertEquals(3, destination.getInt32ToEnumFieldCount()); assertEquals(3, destination.getInt32ToEnumFieldCount());
} }
@ -583,7 +585,7 @@ public class MapTest extends TestCase {
} catch (InvalidProtocolBufferException expected) { } catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap); assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage(); map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty()); assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
} }
map = map =
@ -644,14 +646,14 @@ public class MapTest extends TestCase {
TestOnChangeEventPropagation.Builder parent = TestOnChangeEventPropagation.newBuilder(); TestOnChangeEventPropagation.Builder parent = TestOnChangeEventPropagation.newBuilder();
parent.getOptionalMessageBuilder().putInt32ToInt32Field(1, 2); parent.getOptionalMessageBuilder().putInt32ToInt32Field(1, 2);
TestOnChangeEventPropagation message = parent.build(); TestOnChangeEventPropagation message = parent.build();
assertEquals(2, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); assertEquals(2, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Make a change using nested builder. // Make a change using nested builder.
parent.getOptionalMessageBuilder().putInt32ToInt32Field(1, 3); parent.getOptionalMessageBuilder().putInt32ToInt32Field(1, 3);
// Should be able to observe the change. // Should be able to observe the change.
message = parent.build(); message = parent.build();
assertEquals(3, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); assertEquals(3, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Make another change using mergeFrom() // Make another change using mergeFrom()
TestMap other = TestMap.newBuilder().putInt32ToInt32Field(1, 4).build(); TestMap other = TestMap.newBuilder().putInt32ToInt32Field(1, 4).build();
@ -659,14 +661,14 @@ public class MapTest extends TestCase {
// Should be able to observe the change. // Should be able to observe the change.
message = parent.build(); message = parent.build();
assertEquals(4, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); assertEquals(4, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Make yet another change by clearing the nested builder. // Make yet another change by clearing the nested builder.
parent.getOptionalMessageBuilder().clear(); parent.getOptionalMessageBuilder().clear();
// Should be able to observe the change. // Should be able to observe the change.
message = parent.build(); message = parent.build();
assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size()); assertEquals(0, message.getOptionalMessage().getInt32ToInt32FieldMap().size());
} }
public void testNestedBuilderOnChangeEventPropagationReflection() { public void testNestedBuilderOnChangeEventPropagationReflection() {
@ -683,7 +685,7 @@ public class MapTest extends TestCase {
// Should be able to observe the change. // Should be able to observe the change.
TestOnChangeEventPropagation message = parentBuilder.build(); TestOnChangeEventPropagation message = parentBuilder.build();
assertEquals(1, message.getOptionalMessage().getInt32ToInt32Field().size()); assertEquals(1, message.getOptionalMessage().getInt32ToInt32FieldMap().size());
// Change the entry value. // Change the entry value.
entryBuilder.putInt32ToInt32Field(1, 4); entryBuilder.putInt32ToInt32Field(1, 4);
@ -692,7 +694,7 @@ public class MapTest extends TestCase {
// Should be able to observe the change. // Should be able to observe the change.
message = parentBuilder.build(); message = parentBuilder.build();
assertEquals(4, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); assertEquals(4, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Clear the nested builder. // Clear the nested builder.
testMapBuilder = parentBuilder.getOptionalMessageBuilder(); testMapBuilder = parentBuilder.getOptionalMessageBuilder();
@ -700,7 +702,7 @@ public class MapTest extends TestCase {
// Should be able to observe the change. // Should be able to observe the change.
message = parentBuilder.build(); message = parentBuilder.build();
assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size()); assertEquals(0, message.getOptionalMessage().getInt32ToInt32FieldMap().size());
} }
// The following methods are used to test reflection API. // The following methods are used to test reflection API.
@ -789,8 +791,8 @@ public class MapTest extends TestCase {
builder.clearField(f("int32_to_int32_field")); builder.clearField(f("int32_to_int32_field"));
builder.clearField(f("int32_to_message_field")); builder.clearField(f("int32_to_message_field"));
message = builder.build(); message = builder.build();
assertEquals(0, message.getInt32ToInt32Field().size()); assertEquals(0, message.getInt32ToInt32FieldMap().size());
assertEquals(0, message.getInt32ToMessageField().size()); assertEquals(0, message.getInt32ToMessageFieldMap().size());
// Test setField() // Test setField()
setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44)); setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44));
@ -801,10 +803,10 @@ public class MapTest extends TestCase {
111, MessageValue.newBuilder().setValue(222).build(), 111, MessageValue.newBuilder().setValue(222).build(),
333, MessageValue.newBuilder().setValue(444).build())); 333, MessageValue.newBuilder().setValue(444).build()));
message = builder.build(); message = builder.build();
assertEquals(22, message.getInt32ToInt32Field().get(11).intValue()); assertEquals(22, message.getInt32ToInt32FieldMap().get(11).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(33).intValue()); assertEquals(44, message.getInt32ToInt32FieldMap().get(33).intValue());
assertEquals(222, message.getInt32ToMessageField().get(111).getValue()); assertEquals(222, message.getInt32ToMessageFieldMap().get(111).getValue());
assertEquals(444, message.getInt32ToMessageField().get(333).getValue()); assertEquals(444, message.getInt32ToMessageFieldMap().get(333).getValue());
// Test addRepeatedField // Test addRepeatedField
builder.addRepeatedField( builder.addRepeatedField(
@ -817,8 +819,8 @@ public class MapTest extends TestCase {
555, 555,
MessageValue.newBuilder().setValue(666).build())); MessageValue.newBuilder().setValue(666).build()));
message = builder.build(); message = builder.build();
assertEquals(66, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(66, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(666, message.getInt32ToMessageField().get(555).getValue()); assertEquals(666, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test addRepeatedField (overriding existing values) // Test addRepeatedField (overriding existing values)
builder.addRepeatedField( builder.addRepeatedField(
@ -831,8 +833,8 @@ public class MapTest extends TestCase {
555, 555,
MessageValue.newBuilder().setValue(555).build())); MessageValue.newBuilder().setValue(555).build()));
message = builder.build(); message = builder.build();
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(555, message.getInt32ToMessageField().get(555).getValue()); assertEquals(555, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test setRepeatedField // Test setRepeatedField
for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) { for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) {
@ -846,9 +848,9 @@ public class MapTest extends TestCase {
builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build()); builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build());
} }
message = builder.build(); message = builder.build();
assertEquals(11, message.getInt32ToInt32Field().get(22).intValue()); assertEquals(11, message.getInt32ToInt32FieldMap().get(22).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(44).intValue()); assertEquals(33, message.getInt32ToInt32FieldMap().get(44).intValue());
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
} }
// See additional coverage in TextFormatTest.java. // See additional coverage in TextFormatTest.java.
@ -937,21 +939,21 @@ public class MapTest extends TestCase {
2, 1000)); // unknown value. 2, 1000)); // unknown value.
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(0)); assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(0));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumFieldMap().get(2));
assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, message.getInt32ToEnumFieldValueMap().get(2).intValue());
// Unknown enum values should be preserved after: // Unknown enum values should be preserved after:
// 1. Serialization and parsing. // 1. Serialization and parsing.
// 2. toBuild(). // 2. toBuild().
// 3. mergeFrom(). // 3. mergeFrom().
message = TestMap.parseFrom(message.toByteString()); message = TestMap.parseFrom(message.toByteString());
assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, message.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = message.toBuilder(); builder = message.toBuilder();
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = TestMap.newBuilder().mergeFrom(message); builder = TestMap.newBuilder().mergeFrom(message);
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue()); assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
// hashCode()/equals() should take unknown enum values into account. // hashCode()/equals() should take unknown enum values into account.
builder.putAllInt32ToEnumFieldValue(newMap(2, 1001)); builder.putAllInt32ToEnumFieldValue(newMap(2, 1001));
@ -960,7 +962,7 @@ public class MapTest extends TestCase {
assertFalse(message.equals(message2)); assertFalse(message.equals(message2));
// Unknown values will be converted to UNRECOGNIZED so the resulted enum map // Unknown values will be converted to UNRECOGNIZED so the resulted enum map
// should be the same. // should be the same.
assertEquals(message2.getInt32ToEnumField(), message.getInt32ToEnumField()); assertEquals(message2.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
} }
public void testUnknownEnumValuesInReflectionApi() throws Exception { public void testUnknownEnumValuesInReflectionApi() throws Exception {
@ -991,7 +993,7 @@ public class MapTest extends TestCase {
// Verify that enum values have been successfully updated. // Verify that enum values have been successfully updated.
TestMap message = builder.build(); TestMap message = builder.build();
for (Map.Entry<Integer, Integer> entry : message.getInt32ToEnumFieldValue().entrySet()) { for (Map.Entry<Integer, Integer> entry : message.getInt32ToEnumFieldValueMap().entrySet()) {
assertEquals(data.get(entry.getKey()) + 1, entry.getValue().intValue()); assertEquals(data.get(entry.getKey()) + 1, entry.getValue().intValue());
} }
} }
@ -1002,18 +1004,18 @@ public class MapTest extends TestCase {
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals( assertEquals(
Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32Field().keySet())); Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32FieldMap().keySet()));
} }
public void testGetMap() { public void testGetMap() {
TestMap.Builder builder = TestMap.newBuilder(); TestMap.Builder builder = TestMap.newBuilder();
setMapValuesUsingAccessors(builder); setMapValuesUsingAccessors(builder);
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap()); assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap()); assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap()); assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToEnumFieldValue(), message.getInt32ToEnumFieldValueMap()); assertEquals(message.getInt32ToEnumFieldValueMap(), message.getInt32ToEnumFieldValueMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap()); assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
} }
public void testContains() { public void testContains() {

@ -30,6 +30,7 @@
package com.google.protobuf; package com.google.protobuf;
import static com.google.common.truth.Truth.assertThat;
import static com.google.protobuf.TestUtil.TEST_REQUIRED_INITIALIZED; import static com.google.protobuf.TestUtil.TEST_REQUIRED_INITIALIZED;
import static com.google.protobuf.TestUtil.TEST_REQUIRED_UNINITIALIZED; import static com.google.protobuf.TestUtil.TEST_REQUIRED_UNINITIALIZED;
@ -866,6 +867,12 @@ public class TextFormatTest extends TestCase {
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\341\\210\\264")); assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\341\\210\\264"));
assertEquals("\u1234", TextFormat.unescapeText("\\xe1\\x88\\xb4")); assertEquals("\u1234", TextFormat.unescapeText("\\xe1\\x88\\xb4"));
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\xe1\\x88\\xb4")); assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\xe1\\x88\\xb4"));
assertEquals("\u1234", TextFormat.unescapeText("\\u1234"));
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\u1234"));
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\U00001234"));
assertEquals(
new String(new int[] {0x10437}, 0, 1), TextFormat.unescapeText("\\xf0\\x90\\x90\\xb7"));
assertEquals(bytes(0xf0, 0x90, 0x90, 0xb7), TextFormat.unescapeBytes("\\U00010437"));
// Handling of strings with unescaped Unicode characters > 255. // Handling of strings with unescaped Unicode characters > 255.
final String zh = "\u9999\u6e2f\u4e0a\u6d77\ud84f\udf80\u8c50\u9280\u884c"; final String zh = "\u9999\u6e2f\u4e0a\u6d77\ud84f\udf80\u8c50\u9280\u884c";
@ -893,6 +900,87 @@ public class TextFormatTest extends TestCase {
} catch (TextFormat.InvalidEscapeSequenceException e) { } catch (TextFormat.InvalidEscapeSequenceException e) {
// success // success
} }
try {
TextFormat.unescapeText("\\u");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' with too few hex chars");
}
try {
TextFormat.unescapeText("\\ud800");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\ud800\\u1234");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\udc00");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\ud801\\udc37");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\U1234");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U' with too few hex chars");
}
try {
TextFormat.unescapeText("\\U1234no more hex");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U' with too few hex chars");
}
try {
TextFormat.unescapeText("\\U00110000");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U00110000' is not a valid code point value");
}
try {
TextFormat.unescapeText("\\U0000d801\\U00000dc37");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U0000d801' refers to a surrogate code unit");
}
} }
public void testParseInteger() throws Exception { public void testParseInteger() throws Exception {

@ -174,8 +174,9 @@ public final class Proto2MessageLiteInfoFactory implements MessageInfoFactory {
"fieldRequiredSint6487_", "fieldRequiredSint6487_",
"fieldRequiredGroup88_", "fieldRequiredGroup88_",
}; };
// To update this after a proto change, run protoc on proto2_message_lite.proto and copy over // To update this after a proto change, run blaze build on proto2_message_lite.proto and copy
// the content of the generated buildMessageInfo() method here. // over the String info from the proto2_message_lite_proto-lite-src.jar file in the
// blaze-genfiles directory.
java.lang.String info = java.lang.String info =
"\u0001U\u0001\u0002\u0001XU\u0000 \u0015\u0001\u1000\u0000\u0002\u1001\u0001\u0003" "\u0001U\u0001\u0002\u0001XU\u0000 \u0015\u0001\u1000\u0000\u0002\u1001\u0001\u0003"
+ "\u1002\u0002\u0004\u1003\u0003\u0005\u1004\u0004\u0006\u1005\u0005\u0007\u1006\u0006\b\u1007\u0007" + "\u1002\u0002\u0004\u1003\u0003\u0005\u1004\u0004\u0006\u1005\u0005\u0007\u1006\u0006\b\u1007\u0007"

@ -137,11 +137,19 @@ final class FieldMaskTree {
} }
/** /**
* Remove {@code path} from the tree. * Removes {@code path} from the tree.
* *
* <p>When removing a field path from the tree, all sub-paths will be removed. That is, after * <ul>
* removing "foo.bar" from the tree, "foo.bar.baz" will be removed. Likewise, if the field path to * When removing a field path from the tree:
* remove is a non-exist sub-path, nothing will be changed. * <li>All sub-paths will be removed. That is, after removing "foo.bar" from the tree,
* "foo.bar.baz" will be removed.
* <li>If all children of a node has been removed, the node itself will be removed as well.
* That is, if "foo" only has one child "bar" and "foo.bar" only has one child "baz",
* removing "foo.bar.barz" would remove both "foo" and "foo.bar".
* If "foo" has both "bar" and "qux" as children, removing "foo.bar" would leave the path
* "foo.qux" intact.
* <li>If the field path to remove is a non-exist sub-path, nothing will be changed.
* </ul>
*/ */
@CanIgnoreReturnValue @CanIgnoreReturnValue
FieldMaskTree removeFieldPath(String path) { FieldMaskTree removeFieldPath(String path) {
@ -149,21 +157,33 @@ final class FieldMaskTree {
if (parts.isEmpty()) { if (parts.isEmpty()) {
return this; return this;
} }
Node node = root; removeFieldPath(root, parts, 0);
for (int i = 0; i < parts.size(); i++) {
String key = parts.get(i);
if (!node.children.containsKey(key)) {
// Path does not exist.
return this; return this;
} }
if (i == parts.size() - 1) {
// Remove path. /**
* Removes {@code parts} from {@code node} recursively.
*
* @return a boolean value indicating whether current {@code node} should be removed.
*/
@CanIgnoreReturnValue
private static boolean removeFieldPath(Node node, List<String> parts, int index) {
String key = parts.get(index);
// Base case 1: path not match.
if (!node.children.containsKey(key)) {
return false;
}
// Base case 2: last element in parts.
if (index == parts.size() - 1) {
node.children.remove(key); node.children.remove(key);
return this; return node.children.isEmpty();
} }
node = node.children.get(key); // Recursive remove sub-path.
if (removeFieldPath(node.children.get(key), parts, index + 1)) {
node.children.remove(key);
} }
return this; return node.children.isEmpty();
} }
/** Removes all field paths in {@code mask} from this tree. */ /** Removes all field paths in {@code mask} from this tree. */
@ -187,10 +207,8 @@ final class FieldMaskTree {
return FieldMask.newBuilder().addAllPaths(paths).build(); return FieldMask.newBuilder().addAllPaths(paths).build();
} }
/** /** Gathers all field paths in a sub-tree. */
* Gathers all field paths in a sub-tree. private static void getFieldPaths(Node node, String path, List<String> paths) {
*/
private void getFieldPaths(Node node, String path, List<String> paths) {
if (node.children.isEmpty()) { if (node.children.isEmpty()) {
paths.add(path); paths.add(path);
return; return;
@ -247,10 +265,8 @@ final class FieldMaskTree {
merge(root, "", source, destination, options); merge(root, "", source, destination, options);
} }
/** /** Merges all fields specified by a sub-tree from {@code source} to {@code destination}. */
* Merges all fields specified by a sub-tree from {@code source} to {@code destination}. private static void merge(
*/
private void merge(
Node node, Node node,
String path, String path,
Message source, Message source,

@ -276,7 +276,13 @@ public final class FieldMaskUtil {
return maskTree.toFieldMask(); return maskTree.toFieldMask();
} }
/** Subtracts {@code secondMask} and {@code otherMasks} from {@code firstMask}. */ /**
* Subtracts {@code secondMask} and {@code otherMasks} from {@code firstMask}.
*
* <p>This method disregards proto structure. That is, if {@code firstMask} is "foo" and {@code
* secondMask} is "foo.bar", the response will always be "foo" without considering the internal
* proto structure of message "foo".
*/
public static FieldMask subtract( public static FieldMask subtract(
FieldMask firstMask, FieldMask secondMask, FieldMask... otherMasks) { FieldMask firstMask, FieldMask secondMask, FieldMask... otherMasks) {
FieldMaskTree maskTree = new FieldMaskTree(firstMask).removeFromFieldMask(secondMask); FieldMaskTree maskTree = new FieldMaskTree(firstMask).removeFromFieldMask(secondMask);

@ -525,7 +525,6 @@ public class JsonFormat {
return types.get(name); return types.get(name);
} }
/* @Nullable */
Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException { Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException {
return find(getTypeName(typeUrl)); return find(getTypeName(typeUrl));
} }

@ -77,25 +77,37 @@ public class FieldMaskTreeTest extends TestCase {
public void testRemoveFieldPath() throws Exception { public void testRemoveFieldPath() throws Exception {
String initialTreeString = "bar.baz,bar.quz.bar,foo"; String initialTreeString = "bar.baz,bar.quz.bar,foo";
FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString)); FieldMaskTree tree;
// Empty path. // Empty path.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath(""); tree.removeFieldPath("");
assertEquals(initialTreeString, tree.toString()); assertEquals(initialTreeString, tree.toString());
// Non-exist sub-path of an existing leaf. // Non-exist sub-path of an existing leaf.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("foo.bar"); tree.removeFieldPath("foo.bar");
assertEquals(initialTreeString, tree.toString()); assertEquals(initialTreeString, tree.toString());
// Non-exist path. // Non-exist path.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("bar.foo"); tree.removeFieldPath("bar.foo");
assertEquals(initialTreeString, tree.toString()); assertEquals(initialTreeString, tree.toString());
// Match an existing leaf node.
// Match an existing leaf node -> remove leaf node.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("foo"); tree.removeFieldPath("foo");
assertEquals("bar.baz,bar.quz.bar", tree.toString()); assertEquals("bar.baz,bar.quz.bar", tree.toString());
// Match sub-path of an existing leaf node.
// Match sub-path of an existing leaf node -> recursive removal.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("bar.quz.bar"); tree.removeFieldPath("bar.quz.bar");
assertEquals("bar.baz,bar.quz", tree.toString()); assertEquals("bar.baz,foo", tree.toString());
// Match a non-leaf node.
// Match a non-leaf node -> remove all children.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("bar"); tree.removeFieldPath("bar");
assertThat(tree.toString()).isEmpty(); assertEquals("foo", tree.toString());
} }
public void testRemoveFromFieldMask() throws Exception { public void testRemoveFromFieldMask() throws Exception {

@ -239,7 +239,7 @@ public class FieldMaskUtilTest extends TestCase {
FieldMask mask3 = FieldMaskUtil.fromString("bar.quz"); FieldMask mask3 = FieldMaskUtil.fromString("bar.quz");
FieldMask mask4 = FieldMaskUtil.fromString("foo,bar.baz"); FieldMask mask4 = FieldMaskUtil.fromString("foo,bar.baz");
FieldMask result = FieldMaskUtil.subtract(mask1, mask2, mask3, mask4); FieldMask result = FieldMaskUtil.subtract(mask1, mask2, mask3, mask4);
assertEquals("bar", FieldMaskUtil.toString(result)); assertThat(FieldMaskUtil.toString(result)).isEmpty();
} }
public void testIntersection() throws Exception { public void testIntersection() throws Exception {

@ -722,8 +722,8 @@ public class JsonFormatTest extends TestCase {
mergeFromJson( mergeFromJson(
"{\n" + " int32ToInt32Map: {1: 2},\n" + " stringToInt32Map: {hello: 3}\n" + "}", builder); "{\n" + " int32ToInt32Map: {1: 2},\n" + " stringToInt32Map: {hello: 3}\n" + "}", builder);
TestMap message = builder.build(); TestMap message = builder.build();
assertEquals(2, message.getInt32ToInt32Map().get(1).intValue()); assertEquals(2, message.getInt32ToInt32MapMap().get(1).intValue());
assertEquals(3, message.getStringToInt32Map().get("hello").intValue()); assertEquals(3, message.getStringToInt32MapMap().get("hello").intValue());
} }
public void testWrappers() throws Exception { public void testWrappers() throws Exception {

@ -31,6 +31,7 @@
/** /**
* @fileoverview This file contains constants and typedefs used by * @fileoverview This file contains constants and typedefs used by
* jspb.BinaryReader and BinaryWriter. * jspb.BinaryReader and BinaryWriter.
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* *
* @author aappleby@google.com (Austin Appleby) * @author aappleby@google.com (Austin Appleby)
*/ */

@ -40,6 +40,7 @@
* intact, you _must_ read them using one of the Hash64 methods, which return * intact, you _must_ read them using one of the Hash64 methods, which return
* an 8-character string. * an 8-character string.
* *
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby) * @author aappleby@google.com (Austin Appleby)
*/ */

@ -41,6 +41,7 @@
* using the typed jspb code generator, but if you bypass that you'll need * using the typed jspb code generator, but if you bypass that you'll need
* to keep things in sync by hand. * to keep things in sync by hand.
* *
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby) * @author aappleby@google.com (Austin Appleby)
*/ */

@ -32,6 +32,7 @@
* @fileoverview This file contains helper code used by jspb.BinaryReader * @fileoverview This file contains helper code used by jspb.BinaryReader
* and BinaryWriter. * and BinaryWriter.
* *
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby) * @author aappleby@google.com (Austin Appleby)
*/ */

@ -52,6 +52,7 @@
* Major caveat 3 - This class uses typed arrays and must not be used on older * Major caveat 3 - This class uses typed arrays and must not be used on older
* browsers that do not support them. * browsers that do not support them.
* *
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby) * @author aappleby@google.com (Austin Appleby)
*/ */

@ -1,3 +1,4 @@
// Protocol Buffers - Google's data interchange format // Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved. // Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/ // https://developers.google.com/protocol-buffers/
@ -28,6 +29,10 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/**
* @fileoverview
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
*/
goog.provide('jspb.Map'); goog.provide('jspb.Map');
goog.require('goog.asserts'); goog.require('goog.asserts');

@ -31,6 +31,7 @@
/** /**
* @fileoverview Definition of jspb.Message. * @fileoverview Definition of jspb.Message.
* *
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author mwr@google.com (Mark Rawling) * @author mwr@google.com (Mark Rawling)
*/ */

@ -71,6 +71,8 @@ static NSString *const kGPBDataCoderKey = @"GPBData";
@package @package
GPBUnknownFieldSet *unknownFields_; GPBUnknownFieldSet *unknownFields_;
NSMutableDictionary *extensionMap_; NSMutableDictionary *extensionMap_;
// Readonly access to autocreatedExtensionMap_ is protected via
// readOnlySemaphore_.
NSMutableDictionary *autocreatedExtensionMap_; NSMutableDictionary *autocreatedExtensionMap_;
// If the object was autocreated, we remember the creator so that if we get // If the object was autocreated, we remember the creator so that if we get
@ -79,10 +81,10 @@ static NSString *const kGPBDataCoderKey = @"GPBData";
GPBFieldDescriptor *autocreatorField_; GPBFieldDescriptor *autocreatorField_;
GPBExtensionDescriptor *autocreatorExtension_; GPBExtensionDescriptor *autocreatorExtension_;
// A lock to provide mutual exclusion from internal data that can be modified // Message can only be mutated from one thread. But some *readonly* operations
// by *read* operations such as getters (autocreation of message fields and // modifify internal state because they autocreate things. The
// message extensions, not setting of values). Used to guarantee thread safety // autocreatedExtensionMap_ is one such structure. Access during readonly
// for concurrent reads on the message. // operations is protected via this semaphore.
// NOTE: OSSpinLock may seem like a good fit here but Apple engineers have // NOTE: OSSpinLock may seem like a good fit here but Apple engineers have
// pointed out that they are vulnerable to live locking on iOS in cases of // pointed out that they are vulnerable to live locking on iOS in cases of
// priority inversion: // priority inversion:
@ -583,19 +585,30 @@ static id GetOrCreateArrayIvarWithField(GPBMessage *self,
// This is like GPBGetObjectIvarWithField(), but for arrays, it should // This is like GPBGetObjectIvarWithField(), but for arrays, it should
// only be used to wire the method into the class. // only be used to wire the method into the class.
static id GetArrayIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { static id GetArrayIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) {
id array = GPBGetObjectIvarWithFieldNoAutocreate(self, field); uint8_t *storage = (uint8_t *)self->messageStorage_;
if (!array) { _Atomic(id) *typePtr = (_Atomic(id) *)&storage[field->description_->offset];
// Check again after getting the lock. id array = atomic_load(typePtr);
GPBPrepareReadOnlySemaphore(self); if (array) {
dispatch_semaphore_wait(self->readOnlySemaphore_, DISPATCH_TIME_FOREVER); return array;
array = GPBGetObjectIvarWithFieldNoAutocreate(self, field);
if (!array) {
array = CreateArrayForField(field, self);
GPBSetAutocreatedRetainedObjectIvarWithField(self, field, array);
} }
dispatch_semaphore_signal(self->readOnlySemaphore_);
id expected = nil;
id autocreated = CreateArrayForField(field, self);
if (atomic_compare_exchange_strong(typePtr, &expected, autocreated)) {
// Value was set, return it.
return autocreated;
} }
return array;
// Some other thread set it, release the one created and return what got set.
if (GPBFieldDataTypeIsObject(field)) {
GPBAutocreatedArray *autoArray = autocreated;
autoArray->_autocreator = nil;
} else {
GPBInt32Array *gpbArray = autocreated;
gpbArray->_autocreator = nil;
}
[autocreated release];
return expected;
} }
static id GetOrCreateMapIvarWithField(GPBMessage *self, static id GetOrCreateMapIvarWithField(GPBMessage *self,
@ -613,19 +626,31 @@ static id GetOrCreateMapIvarWithField(GPBMessage *self,
// This is like GPBGetObjectIvarWithField(), but for maps, it should // This is like GPBGetObjectIvarWithField(), but for maps, it should
// only be used to wire the method into the class. // only be used to wire the method into the class.
static id GetMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { static id GetMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) {
id dict = GPBGetObjectIvarWithFieldNoAutocreate(self, field); uint8_t *storage = (uint8_t *)self->messageStorage_;
if (!dict) { _Atomic(id) *typePtr = (_Atomic(id) *)&storage[field->description_->offset];
// Check again after getting the lock. id dict = atomic_load(typePtr);
GPBPrepareReadOnlySemaphore(self); if (dict) {
dispatch_semaphore_wait(self->readOnlySemaphore_, DISPATCH_TIME_FOREVER); return dict;
dict = GPBGetObjectIvarWithFieldNoAutocreate(self, field);
if (!dict) {
dict = CreateMapForField(field, self);
GPBSetAutocreatedRetainedObjectIvarWithField(self, field, dict);
} }
dispatch_semaphore_signal(self->readOnlySemaphore_);
id expected = nil;
id autocreated = CreateMapForField(field, self);
if (atomic_compare_exchange_strong(typePtr, &expected, autocreated)) {
// Value was set, return it.
return autocreated;
} }
return dict;
// Some other thread set it, release the one created and return what got set.
if ((field.mapKeyDataType == GPBDataTypeString) &&
GPBFieldDataTypeIsObject(field)) {
GPBAutocreatedDictionary *autoDict = autocreated;
autoDict->_autocreator = nil;
} else {
GPBInt32Int32Dictionary *gpbDict = autocreated;
gpbDict->_autocreator = nil;
}
[autocreated release];
return expected;
} }
#endif // !defined(__clang_analyzer__) #endif // !defined(__clang_analyzer__)
@ -3337,30 +3362,34 @@ id GPBGetMessageMapField(GPBMessage *self, GPBFieldDescriptor *field) {
id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) {
NSCAssert(!GPBFieldIsMapOrArray(field), @"Shouldn't get here"); NSCAssert(!GPBFieldIsMapOrArray(field), @"Shouldn't get here");
if (!GPBFieldDataTypeIsMessage(field)) {
if (GPBGetHasIvarField(self, field)) { if (GPBGetHasIvarField(self, field)) {
uint8_t *storage = (uint8_t *)self->messageStorage_; uint8_t *storage = (uint8_t *)self->messageStorage_;
id *typePtr = (id *)&storage[field->description_->offset]; id *typePtr = (id *)&storage[field->description_->offset];
return *typePtr; return *typePtr;
} }
// Not set... // Not set...non messages (string/data), get their default.
// Non messages (string/data), get their default.
if (!GPBFieldDataTypeIsMessage(field)) {
return field.defaultValue.valueMessage; return field.defaultValue.valueMessage;
} }
GPBPrepareReadOnlySemaphore(self); uint8_t *storage = (uint8_t *)self->messageStorage_;
dispatch_semaphore_wait(self->readOnlySemaphore_, DISPATCH_TIME_FOREVER); _Atomic(id) *typePtr = (_Atomic(id) *)&storage[field->description_->offset];
GPBMessage *result = GPBGetObjectIvarWithFieldNoAutocreate(self, field); id msg = atomic_load(typePtr);
if (!result) { if (msg) {
// For non repeated messages, create the object, set it and return it. return msg;
// This object will not initially be visible via GPBGetHasIvar, so }
// we save its creator so it can become visible if it's mutated later.
result = GPBCreateMessageWithAutocreator(field.msgClass, self, field); id expected = nil;
GPBSetAutocreatedRetainedObjectIvarWithField(self, field, result); id autocreated = GPBCreateMessageWithAutocreator(field.msgClass, self, field);
} if (atomic_compare_exchange_strong(typePtr, &expected, autocreated)) {
dispatch_semaphore_signal(self->readOnlySemaphore_); // Value was set, return it.
return result; return autocreated;
}
// Some other thread set it, release the one created and return what got set.
GPBClearMessageAutocreator(autocreated);
[autocreated release];
return expected;
} }
#pragma clang diagnostic pop #pragma clang diagnostic pop

@ -504,15 +504,6 @@ static void GPBMaybeClearOneofPrivate(GPBMessage *self,
// Object types are handled slightly differently, they need to be released // Object types are handled slightly differently, they need to be released
// and retained. // and retained.
void GPBSetAutocreatedRetainedObjectIvarWithField(
GPBMessage *self, GPBFieldDescriptor *field,
id __attribute__((ns_consumed)) value) {
uint8_t *storage = (uint8_t *)self->messageStorage_;
id *typePtr = (id *)&storage[field->description_->offset];
NSCAssert(*typePtr == NULL, @"Can't set autocreated object more than once.");
*typePtr = value;
}
void GPBClearAutocreatedMessageIvarWithField(GPBMessage *self, void GPBClearAutocreatedMessageIvarWithField(GPBMessage *self,
GPBFieldDescriptor *field) { GPBFieldDescriptor *field) {
if (GPBGetHasIvarField(self, field)) { if (GPBGetHasIvarField(self, field)) {

@ -289,10 +289,6 @@ void GPBSetRetainedObjectIvarWithFieldPrivate(GPBMessage *self,
id GPBGetObjectIvarWithFieldNoAutocreate(GPBMessage *self, id GPBGetObjectIvarWithFieldNoAutocreate(GPBMessage *self,
GPBFieldDescriptor *field); GPBFieldDescriptor *field);
void GPBSetAutocreatedRetainedObjectIvarWithField(
GPBMessage *self, GPBFieldDescriptor *field,
id __attribute__((ns_consumed)) value);
// Clears and releases the autocreated message ivar, if it's autocreated. If // Clears and releases the autocreated message ivar, if it's autocreated. If
// it's not set as autocreated, this method does nothing. // it's not set as autocreated, this method does nothing.
void GPBClearAutocreatedMessageIvarWithField(GPBMessage *self, void GPBClearAutocreatedMessageIvarWithField(GPBMessage *self,

@ -60,11 +60,16 @@ if _api_version < 0: # Still unspecified?
raise ImportError('_use_fast_cpp_protos import succeeded but was None') raise ImportError('_use_fast_cpp_protos import succeeded but was None')
del _use_fast_cpp_protos del _use_fast_cpp_protos
_api_version = 2 _api_version = 2
# Can not import both use_fast_cpp_protos and use_pure_python.
from google.protobuf import use_pure_python
raise RuntimeError(
'Conflict depend on both use_fast_cpp_protos and use_pure_python')
except ImportError: except ImportError:
try: try:
# pylint: disable=g-import-not-at-top # pylint: disable=g-import-not-at-top
from google.protobuf.internal import use_pure_python from google.protobuf import use_pure_python
del use_pure_python # Avoids a pylint error and namespace pollution. del use_pure_python # Avoids a pylint error and namespace pollution.
_api_version = 0
except ImportError: except ImportError:
# TODO(b/74017912): It's unsafe to enable :use_fast_cpp_protos by default; # TODO(b/74017912): It's unsafe to enable :use_fast_cpp_protos by default;
# it can cause data loss if you have any Python-only extensions to any # it can cause data loss if you have any Python-only extensions to any

@ -629,7 +629,8 @@ class MessageMap(MutableMapping):
return repr(self._values) return repr(self._values)
def MergeFrom(self, other): def MergeFrom(self, other):
for key in other: # pylint: disable=protected-access
for key in other._values:
# According to documentation: "When parsing from the wire or when merging, # According to documentation: "When parsing from the wire or when merging,
# if there are duplicate map keys the last key seen is used". # if there are duplicate map keys the last key seen is used".
if key in self: if key in self:

@ -2107,6 +2107,11 @@ class Proto3Test(unittest.TestCase):
self.assertEqual(msg.map_int32_foreign_message[222].d, 20) self.assertEqual(msg.map_int32_foreign_message[222].d, 20)
self.assertNotEqual(msg.map_int32_foreign_message[222].c, 123) self.assertNotEqual(msg.map_int32_foreign_message[222].c, 123)
# Merge a dict to map field is not accepted
with self.assertRaises(AttributeError):
m1.map_int32_all_types.MergeFrom(
{1: unittest_proto3_arena_pb2.TestAllTypes()})
def testMergeFromBadType(self): def testMergeFromBadType(self):
msg = map_unittest_pb2.TestMap() msg = map_unittest_pb2.TestMap()
with self.assertRaisesRegexp( with self.assertRaisesRegexp(

@ -339,6 +339,11 @@ PyObject* GetEntryClass(PyObject* _self) {
PyObject* MapReflectionFriend::MergeFrom(PyObject* _self, PyObject* arg) { PyObject* MapReflectionFriend::MergeFrom(PyObject* _self, PyObject* arg) {
MapContainer* self = GetMap(_self); MapContainer* self = GetMap(_self);
if (!PyObject_TypeCheck(arg, ScalarMapContainer_Type) &&
!PyObject_TypeCheck(arg, MessageMapContainer_Type)) {
PyErr_SetString(PyExc_AttributeError, "Not a map field");
return nullptr;
}
MapContainer* other_map = GetMap(arg); MapContainer* other_map = GetMap(arg);
Message* message = self->GetMutableMessage(); Message* message = self->GetMutableMessage();
const Message* other_message = other_map->parent->message; const Message* other_message = other_map->parent->message;

@ -809,7 +809,7 @@ bool CheckAndSetString(
return false; return false;
} }
string value_string(value, value_len); std::string value_string(value, value_len);
if (append) { if (append) {
reflection->AddString(message, descriptor, std::move(value_string)); reflection->AddString(message, descriptor, std::move(value_string));
} else if (index < 0) { } else if (index < 0) {

@ -882,8 +882,11 @@ class _Parser(object):
raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %
(expanded_any_end_token,)) (expanded_any_end_token,))
self._MergeField(tokenizer, expanded_any_sub_message) self._MergeField(tokenizer, expanded_any_sub_message)
deterministic = False
message.Pack(expanded_any_sub_message, message.Pack(expanded_any_sub_message,
type_url_prefix=type_url_prefix) type_url_prefix=type_url_prefix,
deterministic=deterministic)
return return
if tokenizer.TryConsume('['): if tokenizer.TryConsume('['):

@ -68,7 +68,6 @@ nobase_include_HEADERS = \
google/protobuf/stubs/bytestream.h \ google/protobuf/stubs/bytestream.h \
google/protobuf/stubs/casts.h \ google/protobuf/stubs/casts.h \
google/protobuf/stubs/common.h \ google/protobuf/stubs/common.h \
google/protobuf/stubs/fastmem.h \
google/protobuf/stubs/hash.h \ google/protobuf/stubs/hash.h \
google/protobuf/stubs/logging.h \ google/protobuf/stubs/logging.h \
google/protobuf/stubs/macros.h \ google/protobuf/stubs/macros.h \
@ -104,7 +103,6 @@ nobase_include_HEADERS = \
google/protobuf/generated_message_util.h \ google/protobuf/generated_message_util.h \
google/protobuf/has_bits.h \ google/protobuf/has_bits.h \
google/protobuf/implicit_weak_message.h \ google/protobuf/implicit_weak_message.h \
google/protobuf/inlined_string_field.h \
google/protobuf/io/io_win32.h \ google/protobuf/io/io_win32.h \
google/protobuf/map_entry.h \ google/protobuf/map_entry.h \
google/protobuf/map_entry_lite.h \ google/protobuf/map_entry_lite.h \
@ -202,6 +200,7 @@ libprotobuf_lite_la_SOURCES = \
google/protobuf/stubs/time.h \ google/protobuf/stubs/time.h \
google/protobuf/any_lite.cc \ google/protobuf/any_lite.cc \
google/protobuf/arena.cc \ google/protobuf/arena.cc \
google/protobuf/arenastring.cc \
google/protobuf/extension_set.cc \ google/protobuf/extension_set.cc \
google/protobuf/generated_enum_util.cc \ google/protobuf/generated_enum_util.cc \
google/protobuf/generated_message_util.cc \ google/protobuf/generated_message_util.cc \

@ -51,9 +51,8 @@ void AnyMetadata::PackFrom(const Message& message,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString(), &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString(),
GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix), GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix),
nullptr); nullptr);
message.SerializeToString(value_->Mutable( message.SerializeToString(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value_->Mutable(ArenaStringPtr::EmptyDefault{}, nullptr));
nullptr));
} }
bool AnyMetadata::UnpackTo(Message* message) const { bool AnyMetadata::UnpackTo(Message* message) const {

@ -60,7 +60,8 @@ class PROTOBUF_EXPORT AnyMetadata {
typedef ArenaStringPtr ValueType; typedef ArenaStringPtr ValueType;
public: public:
// AnyMetadata does not take ownership of "type_url" and "value". // AnyMetadata does not take ownership of "type_url" and "value".
AnyMetadata(UrlType* type_url, ValueType* value); constexpr AnyMetadata(UrlType* type_url, ValueType* value)
: type_url_(type_url), value_(value) {}
// Packs a message using the default type URL prefix: "type.googleapis.com". // Packs a message using the default type URL prefix: "type.googleapis.com".
// The resulted type URL will be "type.googleapis.com/<message_full_name>". // The resulted type URL will be "type.googleapis.com/<message_full_name>".

@ -112,12 +112,12 @@ Any::Any(const Any& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_type_url().empty()) { if (!from._internal_type_url().empty()) {
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_type_url(), type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_type_url(),
GetArena()); GetArena());
} }
value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_value().empty()) { if (!from._internal_value().empty()) {
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_value(), value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_value(),
GetArena()); GetArena());
} }
// @@protoc_insertion_point(copy_constructor:google.protobuf.Any) // @@protoc_insertion_point(copy_constructor:google.protobuf.Any)
@ -162,8 +162,8 @@ void Any::Clear() {
// Prevent compiler warnings about cached_has_bits being unused // Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits; (void) cached_has_bits;
type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); type_url_.ClearToEmpty();
value_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); value_.ClearToEmpty();
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
} }

@ -25,7 +25,6 @@
#include <google/protobuf/arenastring.h> #include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h> #include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h> #include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata_lite.h> #include <google/protobuf/metadata_lite.h>
#include <google/protobuf/generated_message_reflection.h> #include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/message.h> #include <google/protobuf/message.h>
@ -272,7 +271,7 @@ class PROTOBUF_EXPORT Any PROTOBUF_FINAL :
// string type_url = 1; // string type_url = 1;
inline void Any::clear_type_url() { inline void Any::clear_type_url() {
type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); type_url_.ClearToEmpty();
} }
inline const std::string& Any::type_url() const { inline const std::string& Any::type_url() const {
// @@protoc_insertion_point(field_get:google.protobuf.Any.type_url) // @@protoc_insertion_point(field_get:google.protobuf.Any.type_url)
@ -291,31 +290,30 @@ inline const std::string& Any::_internal_type_url() const {
} }
inline void Any::_internal_set_type_url(const std::string& value) { inline void Any::_internal_set_type_url(const std::string& value) {
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Any::set_type_url(std::string&& value) { inline void Any::set_type_url(std::string&& value) {
type_url_.Set( type_url_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url)
} }
inline void Any::set_type_url(const char* value) { inline void Any::set_type_url(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url) // @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url)
} }
inline void Any::set_type_url(const char* value, inline void Any::set_type_url(const char* value,
size_t size) { size_t size) {
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url)
} }
inline std::string* Any::_internal_mutable_type_url() { inline std::string* Any::_internal_mutable_type_url() {
return type_url_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return type_url_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Any::release_type_url() { inline std::string* Any::release_type_url() {
// @@protoc_insertion_point(field_release:google.protobuf.Any.type_url) // @@protoc_insertion_point(field_release:google.protobuf.Any.type_url)
@ -334,7 +332,7 @@ inline void Any::set_allocated_type_url(std::string* type_url) {
// bytes value = 2; // bytes value = 2;
inline void Any::clear_value() { inline void Any::clear_value() {
value_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); value_.ClearToEmpty();
} }
inline const std::string& Any::value() const { inline const std::string& Any::value() const {
// @@protoc_insertion_point(field_get:google.protobuf.Any.value) // @@protoc_insertion_point(field_get:google.protobuf.Any.value)
@ -353,31 +351,30 @@ inline const std::string& Any::_internal_value() const {
} }
inline void Any::_internal_set_value(const std::string& value) { inline void Any::_internal_set_value(const std::string& value) {
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Any::set_value(std::string&& value) { inline void Any::set_value(std::string&& value) {
value_.Set( value_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value)
} }
inline void Any::set_value(const char* value) { inline void Any::set_value(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Any.value) // @@protoc_insertion_point(field_set_char:google.protobuf.Any.value)
} }
inline void Any::set_value(const void* value, inline void Any::set_value(const void* value,
size_t size) { size_t size) {
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value)
} }
inline std::string* Any::_internal_mutable_value() { inline std::string* Any::_internal_mutable_value() {
return value_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return value_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Any::release_value() { inline std::string* Any::release_value() {
// @@protoc_insertion_point(field_release:google.protobuf.Any.value) // @@protoc_insertion_point(field_release:google.protobuf.Any.value)

@ -53,16 +53,13 @@ const char kAnyFullTypeName[] = "google.protobuf.Any";
const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/"; const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/";
AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value)
: type_url_(type_url), value_(value) {}
void AnyMetadata::InternalPackFrom(const MessageLite& message, void AnyMetadata::InternalPackFrom(const MessageLite& message,
StringPiece type_url_prefix, StringPiece type_url_prefix,
StringPiece type_name) { StringPiece type_name) {
type_url_->Set(&::google::protobuf::internal::GetEmptyString(), type_url_->Set(&::google::protobuf::internal::GetEmptyString(),
GetTypeUrl(type_name, type_url_prefix), nullptr); GetTypeUrl(type_name, type_url_prefix), nullptr);
message.SerializeToString(value_->Mutable( message.SerializeToString(
&::google::protobuf::internal::GetEmptyStringAlreadyInited(), nullptr)); value_->Mutable(ArenaStringPtr::EmptyDefault{}, nullptr));
} }
bool AnyMetadata::InternalUnpackTo(StringPiece type_name, bool AnyMetadata::InternalUnpackTo(StringPiece type_name,

@ -33,10 +33,18 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google { namespace google {
namespace protobuf { namespace protobuf {
namespace { namespace {
TEST(AnyMetadataTest, ConstInit) {
PROTOBUF_CONSTINIT static internal::AnyMetadata metadata(nullptr, nullptr);
(void)metadata;
}
TEST(AnyTest, TestPackAndUnpack) { TEST(AnyTest, TestPackAndUnpack) {
protobuf_unittest::TestAny submessage; protobuf_unittest::TestAny submessage;
submessage.set_int32_value(12345); submessage.set_int32_value(12345);

@ -204,12 +204,12 @@ Api::Api(const Api& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_name().empty()) { if (!from._internal_name().empty()) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(), name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_name(),
GetArena()); GetArena());
} }
version_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); version_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_version().empty()) { if (!from._internal_version().empty()) {
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_version(), version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_version(),
GetArena()); GetArena());
} }
if (from._internal_has_source_context()) { if (from._internal_has_source_context()) {
@ -268,8 +268,8 @@ void Api::Clear() {
methods_.Clear(); methods_.Clear();
options_.Clear(); options_.Clear();
mixins_.Clear(); mixins_.Clear();
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); name_.ClearToEmpty();
version_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); version_.ClearToEmpty();
if (GetArena() == nullptr && source_context_ != nullptr) { if (GetArena() == nullptr && source_context_ != nullptr) {
delete source_context_; delete source_context_;
} }
@ -614,17 +614,17 @@ Method::Method(const Method& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_name().empty()) { if (!from._internal_name().empty()) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(), name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_name(),
GetArena()); GetArena());
} }
request_type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); request_type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_request_type_url().empty()) { if (!from._internal_request_type_url().empty()) {
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_request_type_url(), request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_request_type_url(),
GetArena()); GetArena());
} }
response_type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); response_type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_response_type_url().empty()) { if (!from._internal_response_type_url().empty()) {
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_response_type_url(), response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_response_type_url(),
GetArena()); GetArena());
} }
::memcpy(&request_streaming_, &from.request_streaming_, ::memcpy(&request_streaming_, &from.request_streaming_,
@ -679,9 +679,9 @@ void Method::Clear() {
(void) cached_has_bits; (void) cached_has_bits;
options_.Clear(); options_.Clear();
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); name_.ClearToEmpty();
request_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); request_type_url_.ClearToEmpty();
response_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); response_type_url_.ClearToEmpty();
::memset(&request_streaming_, 0, static_cast<size_t>( ::memset(&request_streaming_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&syntax_) - reinterpret_cast<char*>(&syntax_) -
reinterpret_cast<char*>(&request_streaming_)) + sizeof(syntax_)); reinterpret_cast<char*>(&request_streaming_)) + sizeof(syntax_));
@ -1008,12 +1008,12 @@ Mixin::Mixin(const Mixin& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_name().empty()) { if (!from._internal_name().empty()) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(), name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_name(),
GetArena()); GetArena());
} }
root_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); root_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_root().empty()) { if (!from._internal_root().empty()) {
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_root(), root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_root(),
GetArena()); GetArena());
} }
// @@protoc_insertion_point(copy_constructor:google.protobuf.Mixin) // @@protoc_insertion_point(copy_constructor:google.protobuf.Mixin)
@ -1058,8 +1058,8 @@ void Mixin::Clear() {
// Prevent compiler warnings about cached_has_bits being unused // Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits; (void) cached_has_bits;
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); name_.ClearToEmpty();
root_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); root_.ClearToEmpty();
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
} }

@ -25,7 +25,6 @@
#include <google/protobuf/arenastring.h> #include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h> #include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h> #include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata_lite.h> #include <google/protobuf/metadata_lite.h>
#include <google/protobuf/generated_message_reflection.h> #include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/message.h> #include <google/protobuf/message.h>
@ -732,7 +731,7 @@ class PROTOBUF_EXPORT Mixin PROTOBUF_FINAL :
// string name = 1; // string name = 1;
inline void Api::clear_name() { inline void Api::clear_name() {
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); name_.ClearToEmpty();
} }
inline const std::string& Api::name() const { inline const std::string& Api::name() const {
// @@protoc_insertion_point(field_get:google.protobuf.Api.name) // @@protoc_insertion_point(field_get:google.protobuf.Api.name)
@ -751,31 +750,30 @@ inline const std::string& Api::_internal_name() const {
} }
inline void Api::_internal_set_name(const std::string& value) { inline void Api::_internal_set_name(const std::string& value) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Api::set_name(std::string&& value) { inline void Api::set_name(std::string&& value) {
name_.Set( name_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.name) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.name)
} }
inline void Api::set_name(const char* value) { inline void Api::set_name(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Api.name) // @@protoc_insertion_point(field_set_char:google.protobuf.Api.name)
} }
inline void Api::set_name(const char* value, inline void Api::set_name(const char* value,
size_t size) { size_t size) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.name) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.name)
} }
inline std::string* Api::_internal_mutable_name() { inline std::string* Api::_internal_mutable_name() {
return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return name_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Api::release_name() { inline std::string* Api::release_name() {
// @@protoc_insertion_point(field_release:google.protobuf.Api.name) // @@protoc_insertion_point(field_release:google.protobuf.Api.name)
@ -869,7 +867,7 @@ Api::options() const {
// string version = 4; // string version = 4;
inline void Api::clear_version() { inline void Api::clear_version() {
version_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); version_.ClearToEmpty();
} }
inline const std::string& Api::version() const { inline const std::string& Api::version() const {
// @@protoc_insertion_point(field_get:google.protobuf.Api.version) // @@protoc_insertion_point(field_get:google.protobuf.Api.version)
@ -888,31 +886,30 @@ inline const std::string& Api::_internal_version() const {
} }
inline void Api::_internal_set_version(const std::string& value) { inline void Api::_internal_set_version(const std::string& value) {
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Api::set_version(std::string&& value) { inline void Api::set_version(std::string&& value) {
version_.Set( version_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.version) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.version)
} }
inline void Api::set_version(const char* value) { inline void Api::set_version(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Api.version) // @@protoc_insertion_point(field_set_char:google.protobuf.Api.version)
} }
inline void Api::set_version(const char* value, inline void Api::set_version(const char* value,
size_t size) { size_t size) {
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.version) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.version)
} }
inline std::string* Api::_internal_mutable_version() { inline std::string* Api::_internal_mutable_version() {
return version_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return version_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Api::release_version() { inline std::string* Api::release_version() {
// @@protoc_insertion_point(field_release:google.protobuf.Api.version) // @@protoc_insertion_point(field_release:google.protobuf.Api.version)
@ -1071,7 +1068,7 @@ inline void Api::set_syntax(PROTOBUF_NAMESPACE_ID::Syntax value) {
// string name = 1; // string name = 1;
inline void Method::clear_name() { inline void Method::clear_name() {
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); name_.ClearToEmpty();
} }
inline const std::string& Method::name() const { inline const std::string& Method::name() const {
// @@protoc_insertion_point(field_get:google.protobuf.Method.name) // @@protoc_insertion_point(field_get:google.protobuf.Method.name)
@ -1090,31 +1087,30 @@ inline const std::string& Method::_internal_name() const {
} }
inline void Method::_internal_set_name(const std::string& value) { inline void Method::_internal_set_name(const std::string& value) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Method::set_name(std::string&& value) { inline void Method::set_name(std::string&& value) {
name_.Set( name_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.name) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.name)
} }
inline void Method::set_name(const char* value) { inline void Method::set_name(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Method.name) // @@protoc_insertion_point(field_set_char:google.protobuf.Method.name)
} }
inline void Method::set_name(const char* value, inline void Method::set_name(const char* value,
size_t size) { size_t size) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.name) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.name)
} }
inline std::string* Method::_internal_mutable_name() { inline std::string* Method::_internal_mutable_name() {
return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return name_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Method::release_name() { inline std::string* Method::release_name() {
// @@protoc_insertion_point(field_release:google.protobuf.Method.name) // @@protoc_insertion_point(field_release:google.protobuf.Method.name)
@ -1133,7 +1129,7 @@ inline void Method::set_allocated_name(std::string* name) {
// string request_type_url = 2; // string request_type_url = 2;
inline void Method::clear_request_type_url() { inline void Method::clear_request_type_url() {
request_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); request_type_url_.ClearToEmpty();
} }
inline const std::string& Method::request_type_url() const { inline const std::string& Method::request_type_url() const {
// @@protoc_insertion_point(field_get:google.protobuf.Method.request_type_url) // @@protoc_insertion_point(field_get:google.protobuf.Method.request_type_url)
@ -1152,31 +1148,30 @@ inline const std::string& Method::_internal_request_type_url() const {
} }
inline void Method::_internal_set_request_type_url(const std::string& value) { inline void Method::_internal_set_request_type_url(const std::string& value) {
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Method::set_request_type_url(std::string&& value) { inline void Method::set_request_type_url(std::string&& value) {
request_type_url_.Set( request_type_url_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.request_type_url) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.request_type_url)
} }
inline void Method::set_request_type_url(const char* value) { inline void Method::set_request_type_url(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Method.request_type_url) // @@protoc_insertion_point(field_set_char:google.protobuf.Method.request_type_url)
} }
inline void Method::set_request_type_url(const char* value, inline void Method::set_request_type_url(const char* value,
size_t size) { size_t size) {
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.request_type_url) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.request_type_url)
} }
inline std::string* Method::_internal_mutable_request_type_url() { inline std::string* Method::_internal_mutable_request_type_url() {
return request_type_url_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return request_type_url_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Method::release_request_type_url() { inline std::string* Method::release_request_type_url() {
// @@protoc_insertion_point(field_release:google.protobuf.Method.request_type_url) // @@protoc_insertion_point(field_release:google.protobuf.Method.request_type_url)
@ -1215,7 +1210,7 @@ inline void Method::set_request_streaming(bool value) {
// string response_type_url = 4; // string response_type_url = 4;
inline void Method::clear_response_type_url() { inline void Method::clear_response_type_url() {
response_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); response_type_url_.ClearToEmpty();
} }
inline const std::string& Method::response_type_url() const { inline const std::string& Method::response_type_url() const {
// @@protoc_insertion_point(field_get:google.protobuf.Method.response_type_url) // @@protoc_insertion_point(field_get:google.protobuf.Method.response_type_url)
@ -1234,31 +1229,30 @@ inline const std::string& Method::_internal_response_type_url() const {
} }
inline void Method::_internal_set_response_type_url(const std::string& value) { inline void Method::_internal_set_response_type_url(const std::string& value) {
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Method::set_response_type_url(std::string&& value) { inline void Method::set_response_type_url(std::string&& value) {
response_type_url_.Set( response_type_url_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.response_type_url) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.response_type_url)
} }
inline void Method::set_response_type_url(const char* value) { inline void Method::set_response_type_url(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Method.response_type_url) // @@protoc_insertion_point(field_set_char:google.protobuf.Method.response_type_url)
} }
inline void Method::set_response_type_url(const char* value, inline void Method::set_response_type_url(const char* value,
size_t size) { size_t size) {
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.response_type_url) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.response_type_url)
} }
inline std::string* Method::_internal_mutable_response_type_url() { inline std::string* Method::_internal_mutable_response_type_url() {
return response_type_url_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return response_type_url_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Method::release_response_type_url() { inline std::string* Method::release_response_type_url() {
// @@protoc_insertion_point(field_release:google.protobuf.Method.response_type_url) // @@protoc_insertion_point(field_release:google.protobuf.Method.response_type_url)
@ -1357,7 +1351,7 @@ inline void Method::set_syntax(PROTOBUF_NAMESPACE_ID::Syntax value) {
// string name = 1; // string name = 1;
inline void Mixin::clear_name() { inline void Mixin::clear_name() {
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); name_.ClearToEmpty();
} }
inline const std::string& Mixin::name() const { inline const std::string& Mixin::name() const {
// @@protoc_insertion_point(field_get:google.protobuf.Mixin.name) // @@protoc_insertion_point(field_get:google.protobuf.Mixin.name)
@ -1376,31 +1370,30 @@ inline const std::string& Mixin::_internal_name() const {
} }
inline void Mixin::_internal_set_name(const std::string& value) { inline void Mixin::_internal_set_name(const std::string& value) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Mixin::set_name(std::string&& value) { inline void Mixin::set_name(std::string&& value) {
name_.Set( name_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.name) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.name)
} }
inline void Mixin::set_name(const char* value) { inline void Mixin::set_name(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.name) // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.name)
} }
inline void Mixin::set_name(const char* value, inline void Mixin::set_name(const char* value,
size_t size) { size_t size) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.name) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.name)
} }
inline std::string* Mixin::_internal_mutable_name() { inline std::string* Mixin::_internal_mutable_name() {
return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return name_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Mixin::release_name() { inline std::string* Mixin::release_name() {
// @@protoc_insertion_point(field_release:google.protobuf.Mixin.name) // @@protoc_insertion_point(field_release:google.protobuf.Mixin.name)
@ -1419,7 +1412,7 @@ inline void Mixin::set_allocated_name(std::string* name) {
// string root = 2; // string root = 2;
inline void Mixin::clear_root() { inline void Mixin::clear_root() {
root_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); root_.ClearToEmpty();
} }
inline const std::string& Mixin::root() const { inline const std::string& Mixin::root() const {
// @@protoc_insertion_point(field_get:google.protobuf.Mixin.root) // @@protoc_insertion_point(field_get:google.protobuf.Mixin.root)
@ -1438,31 +1431,30 @@ inline const std::string& Mixin::_internal_root() const {
} }
inline void Mixin::_internal_set_root(const std::string& value) { inline void Mixin::_internal_set_root(const std::string& value) {
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena()); root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
} }
inline void Mixin::set_root(std::string&& value) { inline void Mixin::set_root(std::string&& value) {
root_.Set( root_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.root) // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.root)
} }
inline void Mixin::set_root(const char* value) { inline void Mixin::set_root(const char* value) {
GOOGLE_DCHECK(value != nullptr); GOOGLE_DCHECK(value != nullptr);
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value), root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.root) // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.root)
} }
inline void Mixin::set_root(const char* value, inline void Mixin::set_root(const char* value,
size_t size) { size_t size) {
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string( root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena()); reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.root) // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.root)
} }
inline std::string* Mixin::_internal_mutable_root() { inline std::string* Mixin::_internal_mutable_root() {
return root_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); return root_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
} }
inline std::string* Mixin::release_root() { inline std::string* Mixin::release_root() {
// @@protoc_insertion_point(field_release:google.protobuf.Mixin.root) // @@protoc_insertion_point(field_release:google.protobuf.Mixin.root)

@ -47,10 +47,14 @@ static const size_t kMaxCleanupListElements = 64; // 1kB on 64-bit.
namespace google { namespace google {
namespace protobuf { namespace protobuf {
PROTOBUF_EXPORT /*static*/ void* (*const ArenaOptions::kDefaultBlockAlloc)(
size_t) = &::operator new;
namespace internal { namespace internal {
std::atomic<LifecycleId> ArenaImpl::lifecycle_id_generator_; ArenaImpl::CacheAlignedLifecycleIdGenerator ArenaImpl::lifecycle_id_generator_;
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ =
@ -59,78 +63,205 @@ ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
} }
#elif defined(PROTOBUF_USE_DLLS) #elif defined(PROTOBUF_USE_DLLS)
ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = {-1, NULL}; static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = {
0, static_cast<LifecycleIdAtomic>(-1), nullptr};
return thread_cache_; return thread_cache_;
} }
#else #else
PROTOBUF_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, PROTOBUF_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {
NULL}; 0, static_cast<LifecycleIdAtomic>(-1), nullptr};
#endif
void ArenaFree(void* object, size_t size) {
#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation)
::operator delete(object, size);
#else
(void)size;
::operator delete(object);
#endif #endif
}
ArenaImpl::ArenaImpl(const ArenaOptions& options) {
ArenaMetricsCollector* collector = nullptr;
bool record_allocs = false;
if (options.make_metrics_collector != nullptr) {
collector = (*options.make_metrics_collector)();
record_allocs = (collector && collector->RecordAllocs());
}
// Get memory where we can store non-default options if needed.
// Use supplied initial_block if it is large enough.
size_t min_block_size = kOptionsSize + kBlockHeaderSize + kSerialArenaSize;
char* mem = options.initial_block;
size_t mem_size = options.initial_block_size;
GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(mem) & 7, 0);
if (mem == nullptr || mem_size < min_block_size) {
// Supplied initial block is not big enough.
mem_size = std::max(min_block_size, options.start_block_size);
mem = reinterpret_cast<char*>((*options.block_alloc)(mem_size));
}
// Create the special block.
const bool special = true;
const bool user_owned = (mem == options.initial_block);
auto block =
new (mem) SerialArena::Block(mem_size, nullptr, special, user_owned);
void ArenaImpl::Init() { // Options occupy the beginning of the initial block.
lifecycle_id_ = options_ = new (block->Pointer(block->pos())) Options;
lifecycle_id_generator_.fetch_add(1, std::memory_order_relaxed); #ifdef ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(options_, kOptionsSize);
#endif // ADDRESS_SANITIZER
options_->start_block_size = options.start_block_size;
options_->max_block_size = options.max_block_size;
options_->block_alloc = options.block_alloc;
options_->block_dealloc = options.block_dealloc;
options_->metrics_collector = collector;
block->set_pos(block->pos() + kOptionsSize);
Init(record_allocs);
SetInitialBlock(block);
}
void ArenaImpl::Init(bool record_allocs) {
ThreadCache& tc = thread_cache();
auto id = tc.next_lifecycle_id;
constexpr uint64 kInc = ThreadCache::kPerThreadIds * 2;
if (PROTOBUF_PREDICT_FALSE((id & (kInc - 1)) == 0)) {
if (sizeof(lifecycle_id_generator_.id) == 4) {
// 2^32 is dangerous low to guarantee uniqueness. If we start dolling out
// unique id's in ranges of kInc it's unacceptably low. In this case
// we increment by 1. The additional range of kPerThreadIds that are used
// per thread effectively pushes the overflow time from weeks to years
// of continuous running.
id = lifecycle_id_generator_.id.fetch_add(1, std::memory_order_relaxed) *
kInc;
} else {
id =
lifecycle_id_generator_.id.fetch_add(kInc, std::memory_order_relaxed);
}
}
tc.next_lifecycle_id = id + 2;
// We store "record_allocs" in the low bit of lifecycle_id_.
lifecycle_id_ = id | (record_allocs ? 1 : 0);
hint_.store(nullptr, std::memory_order_relaxed); hint_.store(nullptr, std::memory_order_relaxed);
threads_.store(nullptr, std::memory_order_relaxed); threads_.store(nullptr, std::memory_order_relaxed);
space_allocated_.store(0, std::memory_order_relaxed);
}
if (initial_block_) { void ArenaImpl::SetInitialBlock(SerialArena::Block* block) {
// Thread which calls Init() owns the first block. This allows the // Calling thread owns the first block. This allows the single-threaded case
// single-threaded case to allocate on the first block without having to // to allocate on the first block without having to perform atomic operations.
// perform atomic operations. SerialArena* serial = SerialArena::New(block, &thread_cache(), this);
new (initial_block_) Block(options_.initial_block_size, NULL);
SerialArena* serial =
SerialArena::New(initial_block_, &thread_cache(), this);
serial->set_next(NULL); serial->set_next(NULL);
threads_.store(serial, std::memory_order_relaxed); threads_.store(serial, std::memory_order_relaxed);
space_allocated_.store(options_.initial_block_size, space_allocated_.store(block->size(), std::memory_order_relaxed);
std::memory_order_relaxed);
CacheSerialArena(serial); CacheSerialArena(serial);
} else {
space_allocated_.store(0, std::memory_order_relaxed);
}
} }
ArenaImpl::~ArenaImpl() { ArenaImpl::~ArenaImpl() {
// Have to do this in a first pass, because some of the destructors might // Have to do this in a first pass, because some of the destructors might
// refer to memory in other blocks. // refer to memory in other blocks.
CleanupList(); CleanupList();
FreeBlocks();
ArenaMetricsCollector* collector = nullptr;
auto deallocator = &ArenaFree;
if (options_) {
collector = options_->metrics_collector;
deallocator = options_->block_dealloc;
}
PerBlock([deallocator](SerialArena::Block* b) {
#ifdef ADDRESS_SANITIZER
// This memory was provided by the underlying allocator as unpoisoned, so
// return it in an unpoisoned state.
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
#endif // ADDRESS_SANITIZER
if (!b->user_owned()) {
(*deallocator)(b, b->size());
}
});
if (collector) {
collector->OnDestroy(SpaceAllocated());
}
} }
uint64 ArenaImpl::Reset() { uint64 ArenaImpl::Reset() {
if (options_ && options_->metrics_collector) {
options_->metrics_collector->OnReset(SpaceAllocated());
}
// Have to do this in a first pass, because some of the destructors might // Have to do this in a first pass, because some of the destructors might
// refer to memory in other blocks. // refer to memory in other blocks.
CleanupList(); CleanupList();
uint64 space_allocated = FreeBlocks();
Init();
// Discard all blocks except the special block (if present).
uint64 space_allocated = 0;
SerialArena::Block* special_block = nullptr;
auto deallocator = (options_ ? options_->block_dealloc : &ArenaFree);
PerBlock(
[&space_allocated, &special_block, deallocator](SerialArena::Block* b) {
space_allocated += b->size();
#ifdef ADDRESS_SANITIZER
// This memory was provided by the underlying allocator as unpoisoned,
// so return it in an unpoisoned state.
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
#endif // ADDRESS_SANITIZER
if (!b->special()) {
(*deallocator)(b, b->size());
} else {
// Prepare special block for reuse.
// Note: if options_ is present, it occupies the beginning of the
// block and therefore pos is advanced past it.
GOOGLE_DCHECK(special_block == nullptr);
special_block = b;
}
});
Init(record_allocs());
if (special_block != nullptr) {
// next() should still be nullptr since we are using a stack discipline, but
// clear it anyway to reduce fragility.
GOOGLE_DCHECK_EQ(special_block->next(), nullptr);
special_block->clear_next();
special_block->set_pos(kBlockHeaderSize + (options_ ? kOptionsSize : 0));
SetInitialBlock(special_block);
}
return space_allocated; return space_allocated;
} }
ArenaImpl::Block* ArenaImpl::NewBlock(Block* last_block, size_t min_bytes) { std::pair<void*, size_t> ArenaImpl::NewBuffer(size_t last_size,
size_t min_bytes) {
size_t size; size_t size;
if (last_block) { if (last_size != -1) {
// Double the current block size, up to a limit. // Double the current block size, up to a limit.
size = std::min(2 * last_block->size(), options_.max_block_size); auto max_size = options_ ? options_->max_block_size : kDefaultMaxBlockSize;
size = std::min(2 * last_size, max_size);
} else { } else {
size = options_.start_block_size; size = options_ ? options_->start_block_size : kDefaultStartBlockSize;
} }
// Verify that min_bytes + kBlockHeaderSize won't overflow. // Verify that min_bytes + kBlockHeaderSize won't overflow.
GOOGLE_CHECK_LE(min_bytes, std::numeric_limits<size_t>::max() - kBlockHeaderSize); GOOGLE_CHECK_LE(min_bytes, std::numeric_limits<size_t>::max() - kBlockHeaderSize);
size = std::max(size, kBlockHeaderSize + min_bytes); size = std::max(size, kBlockHeaderSize + min_bytes);
void* mem = options_.block_alloc(size); void* mem = options_ ? (*options_->block_alloc)(size) : ::operator new(size);
Block* b = new (mem) Block(size, last_block);
space_allocated_.fetch_add(size, std::memory_order_relaxed); space_allocated_.fetch_add(size, std::memory_order_relaxed);
return b; return {mem, size};
} }
ArenaImpl::Block::Block(size_t size, Block* next) SerialArena::Block* SerialArena::NewBlock(SerialArena::Block* last_block,
: next_(next), pos_(kBlockHeaderSize), size_(size) {} size_t min_bytes, ArenaImpl* arena) {
void* mem;
size_t size;
std::tie(mem, size) =
arena->NewBuffer(last_block ? last_block->size() : -1, min_bytes);
Block* b = new (mem) Block(size, last_block, false, false);
return b;
}
PROTOBUF_NOINLINE PROTOBUF_NOINLINE
void ArenaImpl::SerialArena::AddCleanupFallback(void* elem, void SerialArena::AddCleanupFallback(void* elem, void (*cleanup)(void*)) {
void (*cleanup)(void*)) {
size_t size = cleanup_ ? cleanup_->size * 2 : kMinCleanupListElements; size_t size = cleanup_ ? cleanup_->size * 2 : kMinCleanupListElements;
size = std::min(size, kMaxCleanupListElements); size = std::min(size, kMaxCleanupListElements);
size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size)); size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size));
@ -182,11 +313,11 @@ void ArenaImpl::AddCleanupFallback(void* elem, void (*cleanup)(void*)) {
} }
PROTOBUF_NOINLINE PROTOBUF_NOINLINE
void* ArenaImpl::SerialArena::AllocateAlignedFallback(size_t n) { void* SerialArena::AllocateAlignedFallback(size_t n) {
// Sync back to current's pos. // Sync back to current's pos.
head_->set_pos(head_->size() - (limit_ - ptr_)); head_->set_pos(head_->size() - (limit_ - ptr_));
head_ = arena_->NewBlock(head_, n); head_ = NewBlock(head_, n, arena_);
ptr_ = head_->Pointer(head_->pos()); ptr_ = head_->Pointer(head_->pos());
limit_ = head_->Pointer(head_->size()); limit_ = head_->Pointer(head_->size());
@ -207,10 +338,14 @@ uint64 ArenaImpl::SpaceUsed() const {
for (; serial; serial = serial->next()) { for (; serial; serial = serial->next()) {
space_used += serial->SpaceUsed(); space_used += serial->SpaceUsed();
} }
// Remove the overhead of Options structure, if any.
if (options_) {
space_used -= kOptionsSize;
}
return space_used; return space_used;
} }
uint64 ArenaImpl::SerialArena::SpaceUsed() const { uint64 SerialArena::SpaceUsed() const {
// Get current block's size from ptr_ (since we can't trust head_->pos(). // Get current block's size from ptr_ (since we can't trust head_->pos().
uint64 space_used = ptr_ - head_->Pointer(kBlockHeaderSize); uint64 space_used = ptr_ - head_->Pointer(kBlockHeaderSize);
// Get subsequent block size from b->pos(). // Get subsequent block size from b->pos().
@ -218,57 +353,10 @@ uint64 ArenaImpl::SerialArena::SpaceUsed() const {
space_used += (b->pos() - kBlockHeaderSize); space_used += (b->pos() - kBlockHeaderSize);
} }
// Remove the overhead of the SerialArena itself. // Remove the overhead of the SerialArena itself.
space_used -= kSerialArenaSize; space_used -= ArenaImpl::kSerialArenaSize;
return space_used; return space_used;
} }
uint64 ArenaImpl::FreeBlocks() {
uint64 space_allocated = 0;
// By omitting an Acquire barrier we ensure that any user code that doesn't
// properly synchronize Reset() or the destructor will throw a TSAN warning.
SerialArena* serial = threads_.load(std::memory_order_relaxed);
while (serial) {
// This is inside a block we are freeing, so we need to read it now.
SerialArena* next = serial->next();
space_allocated += ArenaImpl::SerialArena::Free(serial, initial_block_,
options_.block_dealloc);
// serial is dead now.
serial = next;
}
return space_allocated;
}
uint64 ArenaImpl::SerialArena::Free(ArenaImpl::SerialArena* serial,
Block* initial_block,
void (*block_dealloc)(void*, size_t)) {
uint64 space_allocated = 0;
// We have to be careful in this function, since we will be freeing the Block
// that contains this SerialArena. Be careful about accessing |serial|.
for (Block* b = serial->head_; b;) {
// This is inside the block we are freeing, so we need to read it now.
Block* next_block = b->next();
space_allocated += (b->size());
#ifdef ADDRESS_SANITIZER
// This memory was provided by the underlying allocator as unpoisoned, so
// return it in an unpoisoned state.
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
#endif // ADDRESS_SANITIZER
if (b != initial_block) {
block_dealloc(b, b->size());
}
b = next_block;
}
return space_allocated;
}
void ArenaImpl::CleanupList() { void ArenaImpl::CleanupList() {
// By omitting an Acquire barrier we ensure that any user code that doesn't // By omitting an Acquire barrier we ensure that any user code that doesn't
// properly synchronize Reset() or the destructor will throw a TSAN warning. // properly synchronize Reset() or the destructor will throw a TSAN warning.
@ -279,13 +367,13 @@ void ArenaImpl::CleanupList() {
} }
} }
void ArenaImpl::SerialArena::CleanupList() { void SerialArena::CleanupList() {
if (cleanup_ != NULL) { if (cleanup_ != NULL) {
CleanupListFallback(); CleanupListFallback();
} }
} }
void ArenaImpl::SerialArena::CleanupListFallback() { void SerialArena::CleanupListFallback() {
// The first chunk might be only partially full, so calculate its size // The first chunk might be only partially full, so calculate its size
// from cleanup_ptr_. Subsequent chunks are always full, so use list->size. // from cleanup_ptr_. Subsequent chunks are always full, so use list->size.
size_t n = cleanup_ptr_ - &cleanup_->nodes[0]; size_t n = cleanup_ptr_ - &cleanup_->nodes[0];
@ -305,13 +393,11 @@ void ArenaImpl::SerialArena::CleanupListFallback() {
} }
} }
ArenaImpl::SerialArena* ArenaImpl::SerialArena::New(Block* b, void* owner, SerialArena* SerialArena::New(Block* b, void* owner, ArenaImpl* arena) {
ArenaImpl* arena) { auto pos = b->pos();
GOOGLE_DCHECK_EQ(b->pos(), kBlockHeaderSize); // Should be a fresh block GOOGLE_DCHECK_LE(pos + ArenaImpl::kSerialArenaSize, b->size());
GOOGLE_DCHECK_LE(kBlockHeaderSize + kSerialArenaSize, b->size()); SerialArena* serial = reinterpret_cast<SerialArena*>(b->Pointer(pos));
SerialArena* serial = b->set_pos(pos + ArenaImpl::kSerialArenaSize);
reinterpret_cast<SerialArena*>(b->Pointer(kBlockHeaderSize));
b->set_pos(kBlockHeaderSize + kSerialArenaSize);
serial->arena_ = arena; serial->arena_ = arena;
serial->owner_ = owner; serial->owner_ = owner;
serial->head_ = b; serial->head_ = b;
@ -324,7 +410,7 @@ ArenaImpl::SerialArena* ArenaImpl::SerialArena::New(Block* b, void* owner,
} }
PROTOBUF_NOINLINE PROTOBUF_NOINLINE
ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) { SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
// Look for this SerialArena in our linked list. // Look for this SerialArena in our linked list.
SerialArena* serial = threads_.load(std::memory_order_acquire); SerialArena* serial = threads_.load(std::memory_order_acquire);
for (; serial; serial = serial->next()) { for (; serial; serial = serial->next()) {
@ -336,7 +422,7 @@ ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
if (!serial) { if (!serial) {
// This thread doesn't have any SerialArena, which also means it doesn't // This thread doesn't have any SerialArena, which also means it doesn't
// have any blocks yet. So we'll allocate its first block now. // have any blocks yet. So we'll allocate its first block now.
Block* b = NewBlock(NULL, kSerialArenaSize); SerialArena::Block* b = SerialArena::NewBlock(NULL, kSerialArenaSize, this);
serial = SerialArena::New(b, me, this); serial = SerialArena::New(b, me, this);
SerialArena* head = threads_.load(std::memory_order_relaxed); SerialArena* head = threads_.load(std::memory_order_relaxed);
@ -350,6 +436,8 @@ ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
return serial; return serial;
} }
ArenaMetricsCollector::~ArenaMetricsCollector() {}
} // namespace internal } // namespace internal
PROTOBUF_FUNC_ALIGN(32) PROTOBUF_FUNC_ALIGN(32)
@ -357,25 +445,5 @@ void* Arena::AllocateAlignedNoHook(size_t n) {
return impl_.AllocateAligned(n); return impl_.AllocateAligned(n);
} }
void Arena::CallDestructorHooks() {
uint64 space_allocated = impl_.SpaceAllocated();
// Call the reset hook
if (on_arena_reset_ != NULL) {
on_arena_reset_(this, hooks_cookie_, space_allocated);
}
// Call the destruction hook
if (on_arena_destruction_ != NULL) {
on_arena_destruction_(this, hooks_cookie_, space_allocated);
}
}
void Arena::OnArenaAllocation(const std::type_info* allocated_type,
size_t n) const {
if (on_arena_allocation_ != NULL) {
on_arena_allocation_(allocated_type, n, hooks_cookie_);
}
}
} // namespace protobuf } // namespace protobuf
} // namespace google } // namespace google

@ -102,15 +102,6 @@ template <typename T>
void arena_delete_object(void* object) { void arena_delete_object(void* object) {
delete reinterpret_cast<T*>(object); delete reinterpret_cast<T*>(object);
} }
inline void arena_free(void* object, size_t size) {
#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation)
::operator delete(object, size);
#else
(void)size;
::operator delete(object);
#endif
}
} // namespace internal } // namespace internal
// ArenaOptions provides optional additional parameters to arena construction // ArenaOptions provides optional additional parameters to arena construction
@ -151,44 +142,30 @@ struct ArenaOptions {
max_block_size(kDefaultMaxBlockSize), max_block_size(kDefaultMaxBlockSize),
initial_block(NULL), initial_block(NULL),
initial_block_size(0), initial_block_size(0),
block_alloc(&::operator new), block_alloc(kDefaultBlockAlloc),
block_dealloc(&internal::arena_free), block_dealloc(&internal::ArenaFree),
on_arena_init(NULL), make_metrics_collector(nullptr) {}
on_arena_reset(NULL),
on_arena_destruction(NULL), PROTOBUF_EXPORT static void* (*const kDefaultBlockAlloc)(size_t);
on_arena_allocation(NULL) {}
private: private:
// Hooks for adding external functionality such as user-specific metrics // If make_metrics_collector is not nullptr, it will be called at Arena init
// collection, specific debugging abilities, etc. // time. It may return a pointer to a collector instance that will be notified
// Init hook (if set) will always be called at Arena init time. Init hook may // of interesting events related to the arena.
// return a pointer to a cookie to be stored in the arena. Reset and internal::ArenaMetricsCollector* (*make_metrics_collector)();
// destruction hooks will then be called with the same cookie pointer. This
// allows us to save an external object per arena instance and use it on the
// other hooks (Note: If init hook returns NULL, the other hooks will NOT be
// called on this arena instance).
// on_arena_reset and on_arena_destruction also receive the space used in the
// arena just before the reset.
void* (*on_arena_init)(Arena* arena);
void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used);
void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used);
// type_info is promised to be static - its lifetime extends to
// match program's lifetime (It is given by typeid operator).
// Note: typeid(void) will be passed as allocated_type every time we
// intentionally want to avoid monitoring an allocation. (i.e. internal
// allocations for managing the arena)
void (*on_arena_allocation)(const std::type_info* allocated_type,
uint64 alloc_size, void* cookie);
// Constants define default starting block size and max block size for // Constants define default starting block size and max block size for
// arena allocator behavior -- see descriptions above. // arena allocator behavior -- see descriptions above.
static const size_t kDefaultStartBlockSize = 256; static const size_t kDefaultStartBlockSize =
static const size_t kDefaultMaxBlockSize = 8192; internal::ArenaImpl::kDefaultStartBlockSize;
static const size_t kDefaultMaxBlockSize =
internal::ArenaImpl::kDefaultMaxBlockSize;
friend void arena_metrics::EnableArenaMetrics(ArenaOptions*); friend void arena_metrics::EnableArenaMetrics(ArenaOptions*);
friend class Arena; friend class Arena;
friend class ArenaOptionsTestFriend; friend class ArenaOptionsTestFriend;
friend class internal::ArenaImpl;
}; };
// Support for non-RTTI environments. (The metrics hooks API uses type // Support for non-RTTI environments. (The metrics hooks API uses type
@ -246,11 +223,20 @@ struct ArenaOptions {
// should not rely on this protocol. // should not rely on this protocol.
class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
public: public:
// Arena constructor taking custom options. See ArenaOptions below for // Default constructor with sensible default options, tuned for average
// use-cases.
inline Arena() : impl_() {}
// Construct an arena with default options, except for the supplied
// initial block. It is more efficient to use this constructor
// instead of passing ArenaOptions if the only configuration needed
// by the caller is supplying an initial block.
inline Arena(char* initial_block, size_t initial_block_size)
: impl_(initial_block, initial_block_size) {}
// Arena constructor taking custom options. See ArenaOptions above for
// descriptions of the options available. // descriptions of the options available.
explicit Arena(const ArenaOptions& options) : impl_(options) { explicit Arena(const ArenaOptions& options) : impl_(options) {}
Init(options);
}
// Block overhead. Use this as a guide for how much to over-allocate the // Block overhead. Use this as a guide for how much to over-allocate the
// initial block if you want an allocation of size N to fit inside it. // initial block if you want an allocation of size N to fit inside it.
@ -261,27 +247,10 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
static const size_t kBlockOverhead = internal::ArenaImpl::kBlockHeaderSize + static const size_t kBlockOverhead = internal::ArenaImpl::kBlockHeaderSize +
internal::ArenaImpl::kSerialArenaSize; internal::ArenaImpl::kSerialArenaSize;
// Default constructor with sensible default options, tuned for average inline ~Arena() {}
// use-cases.
Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); }
~Arena() {
if (hooks_cookie_) {
CallDestructorHooks();
}
}
void Init(const ArenaOptions& options) { // TODO(protobuf-team): Fix callers to use constructor and delete this method.
on_arena_allocation_ = options.on_arena_allocation; void Init(const ArenaOptions&) {}
on_arena_reset_ = options.on_arena_reset;
on_arena_destruction_ = options.on_arena_destruction;
// Call the initialization hook
if (options.on_arena_init != NULL) {
hooks_cookie_ = options.on_arena_init(this);
} else {
hooks_cookie_ = NULL;
}
}
// API to create proto2 message objects on the arena. If the arena passed in // API to create proto2 message objects on the arena. If the arena passed in
// is NULL, then a heap allocated object is returned. Type T must be a message // is NULL, then a heap allocated object is returned. Type T must be a message
@ -362,13 +331,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// Any objects allocated on this arena are unusable after this call. It also // Any objects allocated on this arena are unusable after this call. It also
// returns the total space used by the arena which is the sums of the sizes // returns the total space used by the arena which is the sums of the sizes
// of the allocated blocks. This method is not thread-safe. // of the allocated blocks. This method is not thread-safe.
PROTOBUF_NOINLINE uint64 Reset() { uint64 Reset() { return impl_.Reset(); }
// Call the reset hook
if (on_arena_reset_ != NULL) {
on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated());
}
return impl_.Reset();
}
// Adds |object| to a list of heap-allocated objects to be freed with |delete| // Adds |object| to a list of heap-allocated objects to be freed with |delete|
// when the arena is destroyed or reset. // when the arena is destroyed or reset.
@ -515,28 +478,31 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
} }
} }
void CallDestructorHooks();
void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const;
inline void AllocHook(const std::type_info* allocated_type, size_t n) const { inline void AllocHook(const std::type_info* allocated_type, size_t n) const {
if (PROTOBUF_PREDICT_FALSE(hooks_cookie_ != NULL)) { impl_.RecordAlloc(allocated_type, n);
OnArenaAllocation(allocated_type, n);
}
} }
// Allocate and also optionally call on_arena_allocation callback with the // Allocate and also optionally call collector with the allocated type info
// allocated type info when the hooks are in place in ArenaOptions and // when allocation recording is enabled.
// the cookie is not null.
template <typename T> template <typename T>
PROTOBUF_ALWAYS_INLINE void* AllocateInternal(bool skip_explicit_ownership) { PROTOBUF_ALWAYS_INLINE void* AllocateInternal(bool skip_explicit_ownership) {
static_assert(alignof(T) <= 8, "T is overaligned, see b/151247138");
const size_t n = internal::AlignUpTo8(sizeof(T)); const size_t n = internal::AlignUpTo8(sizeof(T));
AllocHook(RTTI_TYPE_ID(T), n);
// Monitor allocation if needed. // Monitor allocation if needed.
impl_.RecordAlloc(RTTI_TYPE_ID(T), n);
if (skip_explicit_ownership) { if (skip_explicit_ownership) {
return AllocateAlignedNoHook(n); return AllocateAlignedTo<alignof(T)>(sizeof(T));
} else { } else {
if (alignof(T) <= 8) {
return impl_.AllocateAlignedAndAddCleanup( return impl_.AllocateAlignedAndAddCleanup(
n, &internal::arena_destruct_object<T>); n, &internal::arena_destruct_object<T>);
} else {
auto ptr =
reinterpret_cast<uintptr_t>(impl_.AllocateAlignedAndAddCleanup(
sizeof(T) + alignof(T) - 8,
&internal::arena_destruct_object<T>));
return reinterpret_cast<void*>((ptr + alignof(T) - 8) &
(~alignof(T) + 1));
}
} }
} }
@ -591,10 +557,12 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
PROTOBUF_ALWAYS_INLINE T* CreateInternalRawArray(size_t num_elements) { PROTOBUF_ALWAYS_INLINE T* CreateInternalRawArray(size_t num_elements) {
GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T)) GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
<< "Requested size is too large to fit into size_t."; << "Requested size is too large to fit into size_t.";
// We count on compiler to realize that if sizeof(T) is a multiple of
// 8 AlignUpTo can be elided.
const size_t n = internal::AlignUpTo8(sizeof(T) * num_elements); const size_t n = internal::AlignUpTo8(sizeof(T) * num_elements);
// Monitor allocation if needed. // Monitor allocation if needed.
AllocHook(RTTI_TYPE_ID(T), n); impl_.RecordAlloc(RTTI_TYPE_ID(T), n);
return static_cast<T*>(AllocateAlignedNoHook(n)); return static_cast<T*>(AllocateAlignedTo<alignof(T)>(n));
} }
template <typename T, typename... Args> template <typename T, typename... Args>
@ -687,7 +655,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// For friends of arena. // For friends of arena.
void* AllocateAligned(size_t n) { void* AllocateAligned(size_t n) {
AllocHook(NULL, n);
return AllocateAlignedNoHook(internal::AlignUpTo8(n)); return AllocateAlignedNoHook(internal::AlignUpTo8(n));
} }
template<size_t Align> template<size_t Align>
@ -698,7 +665,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// TODO(b/151247138): if the pointer would have been aligned already, // TODO(b/151247138): if the pointer would have been aligned already,
// this is wasting space. We should pass the alignment down. // this is wasting space. We should pass the alignment down.
uintptr_t ptr = reinterpret_cast<uintptr_t>(AllocateAligned(n + Align - 8)); uintptr_t ptr = reinterpret_cast<uintptr_t>(AllocateAligned(n + Align - 8));
ptr = (ptr + Align - 1) & -Align; ptr = (ptr + Align - 1) & (~Align + 1);
return reinterpret_cast<void*>(ptr); return reinterpret_cast<void*>(ptr);
} }
@ -706,15 +673,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
internal::ArenaImpl impl_; internal::ArenaImpl impl_;
void (*on_arena_allocation_)(const std::type_info* allocated_type,
uint64 alloc_size, void* cookie);
void (*on_arena_reset_)(Arena* arena, void* cookie, uint64 space_used);
void (*on_arena_destruction_)(Arena* arena, void* cookie, uint64 space_used);
// The arena may save a cookie it receives from the external on_init hook
// and then use it when calling the on_reset and on_destruction hooks.
void* hooks_cookie_;
template <typename Type> template <typename Type>
friend class internal::GenericTypeHandler; friend class internal::GenericTypeHandler;
friend struct internal::ArenaStringPtr; // For AllocateAligned. friend struct internal::ArenaStringPtr; // For AllocateAligned.

@ -48,6 +48,9 @@
namespace google { namespace google {
namespace protobuf { namespace protobuf {
struct ArenaOptions;
namespace internal { namespace internal {
inline size_t AlignUpTo8(size_t n) { inline size_t AlignUpTo8(size_t n) {
@ -55,114 +58,91 @@ inline size_t AlignUpTo8(size_t n) {
return (n + 7) & static_cast<size_t>(-8); return (n + 7) & static_cast<size_t>(-8);
} }
using LifecycleId = int64_t; using LifecycleIdAtomic = uint64_t;
// This class provides the core Arena memory allocation library. Different void PROTOBUF_EXPORT ArenaFree(void* object, size_t size);
// implementations only need to implement the public interface below.
// Arena is not a template type as that would only be useful if all protos // MetricsCollector collects stats for a particular arena.
// in turn would be templates, which will/cannot happen. However separating class PROTOBUF_EXPORT ArenaMetricsCollector {
// the memory allocation part from the cruft of the API users expect we can
// use #ifdef the select the best implementation based on hardware / OS.
class PROTOBUF_EXPORT ArenaImpl {
public: public:
struct Options { virtual ~ArenaMetricsCollector();
size_t start_block_size;
size_t max_block_size; // Invoked when the arena is about to be destroyed. This method will
char* initial_block; // typically finalize any metric collection and delete the collector.
size_t initial_block_size; // space_allocated is the space used by the arena.
void* (*block_alloc)(size_t); virtual void OnDestroy(uint64 space_allocated) = 0;
void (*block_dealloc)(void*, size_t);
// OnReset() is called when the associated arena is reset.
// space_allocated is the space used by the arena just before the reset.
virtual void OnReset(uint64 space_allocated) = 0;
// Does OnAlloc() need to be called? If false, metric collection overhead
// will be reduced since we will not do extra work per allocation.
virtual bool RecordAllocs() = 0;
// OnAlloc is called when an allocation happens.
// type_info is promised to be static - its lifetime extends to
// match program's lifetime (It is given by typeid operator).
// Note: typeid(void) will be passed as allocated_type every time we
// intentionally want to avoid monitoring an allocation. (i.e. internal
// allocations for managing the arena)
virtual void OnAlloc(const std::type_info* allocated_type,
uint64 alloc_size) = 0;
};
template <typename O> class ArenaImpl;
explicit Options(const O& options)
: start_block_size(options.start_block_size),
max_block_size(options.max_block_size),
initial_block(options.initial_block),
initial_block_size(options.initial_block_size),
block_alloc(options.block_alloc),
block_dealloc(options.block_dealloc) {}
};
template <typename O> // A thread-unsafe Arena that can only be used within its owning thread.
explicit ArenaImpl(const O& options) : options_(options) { class PROTOBUF_EXPORT SerialArena {
if (options_.initial_block != NULL && options_.initial_block_size > 0) { public:
GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) // Blocks are variable length malloc-ed objects. The following structure
<< ": Initial block size too small for header."; // describes the common header for all blocks.
initial_block_ = reinterpret_cast<Block*>(options_.initial_block); class PROTOBUF_EXPORT Block {
} else { public:
initial_block_ = NULL; Block(size_t size, Block* next, bool special, bool user_owned)
: next_and_bits_(reinterpret_cast<uintptr_t>(next) | (special ? 1 : 0) |
(user_owned ? 2 : 0)),
pos_(kBlockHeaderSize),
size_(size) {
GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(next) & 3, 0u);
} }
Init(); char* Pointer(size_t n) {
GOOGLE_DCHECK(n <= size_);
return reinterpret_cast<char*>(this) + n;
} }
// Destructor deletes all owned heap allocated objects, and destructs objects // One of the blocks may be special. This is either a user-supplied
// that have non-trivial destructors, except for proto2 message objects whose // initial block, or a block we created at startup to hold Options info.
// destructors can be skipped. Also, frees all blocks except the initial block // A special block is not deleted by Reset.
// if it was passed in. bool special() const { return (next_and_bits_ & 1) != 0; }
~ArenaImpl();
uint64 Reset();
uint64 SpaceAllocated() const; // Whether or not this current block is owned by the user.
uint64 SpaceUsed() const; // Only special blocks can be user_owned.
bool user_owned() const { return (next_and_bits_ & 2) != 0; }
void* AllocateAligned(size_t n) { Block* next() const {
SerialArena* arena; const uintptr_t bottom_bits = 3;
if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { return reinterpret_cast<Block*>(next_and_bits_ & ~bottom_bits);
return arena->AllocateAligned(n);
} else {
return AllocateAlignedFallback(n);
}
} }
// This function allocates n bytes if the common happy case is true and void clear_next() {
// returns true. Otherwise does nothing and returns false. This strange next_and_bits_ &= 3; // Set next to nullptr, preserve bottom bits.
// semantics is necessary to allow callers to program functions that only
// have fallback function calls in tail position. This substantially improves
// code for the happy path.
PROTOBUF_ALWAYS_INLINE bool MaybeAllocateAligned(size_t n, void** out) {
SerialArena* a;
if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFromThreadCache(&a))) {
return a->MaybeAllocateAligned(n, out);
} }
return false;
}
void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*));
// Add object pointer and cleanup function pointer to the list. size_t pos() const { return pos_; }
void AddCleanup(void* elem, void (*cleanup)(void*)); size_t size() const { return size_; }
void set_pos(size_t pos) { pos_ = pos; }
private: private:
friend class ArenaBenchmark; // Holds pointer to next block for this thread + special/user_owned bits.
uintptr_t next_and_bits_;
void* AllocateAlignedFallback(size_t n);
void* AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*));
void AddCleanupFallback(void* elem, void (*cleanup)(void*));
// Node contains the ptr of the object to be cleaned up and the associated
// cleanup function ptr.
struct CleanupNode {
void* elem; // Pointer to the object to be cleaned up.
void (*cleanup)(void*); // Function pointer to the destructor or deleter.
};
// Cleanup uses a chunked linked list, to reduce pointer chasing. size_t pos_;
struct CleanupChunk { size_t size_;
static size_t SizeOf(size_t i) { // data follows
return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1));
}
size_t size; // Total elements in the list.
CleanupChunk* next; // Next node in the list.
CleanupNode nodes[1]; // True length is |size|.
}; };
class Block;
// A thread-unsafe Arena that can only be used within its owning thread.
class PROTOBUF_EXPORT SerialArena {
public:
// The allocate/free methods here are a little strange, since SerialArena is // The allocate/free methods here are a little strange, since SerialArena is
// allocated inside a Block which it also manages. This is to avoid doing // allocated inside a Block which it also manages. This is to avoid doing
// an extra allocation for the SerialArena itself. // an extra allocation for the SerialArena itself.
@ -170,11 +150,6 @@ class PROTOBUF_EXPORT ArenaImpl {
// Creates a new SerialArena inside Block* and returns it. // Creates a new SerialArena inside Block* and returns it.
static SerialArena* New(Block* b, void* owner, ArenaImpl* arena); static SerialArena* New(Block* b, void* owner, ArenaImpl* arena);
// Destroys this SerialArena, freeing all blocks with the given dealloc
// function, except any block equal to |initial_block|.
static uint64 Free(SerialArena* serial, Block* initial_block,
void (*block_dealloc)(void*, size_t));
void CleanupList(); void CleanupList();
uint64 SpaceUsed() const; uint64 SpaceUsed() const;
@ -224,14 +199,29 @@ class PROTOBUF_EXPORT ArenaImpl {
return ret; return ret;
} }
Block* head() const { return head_; }
void* owner() const { return owner_; } void* owner() const { return owner_; }
SerialArena* next() const { return next_; } SerialArena* next() const { return next_; }
void set_next(SerialArena* next) { next_ = next; } void set_next(SerialArena* next) { next_ = next; }
static Block* NewBlock(Block* last_block, size_t min_bytes, ArenaImpl* arena);
private: private:
void* AllocateAlignedFallback(size_t n); // Node contains the ptr of the object to be cleaned up and the associated
void AddCleanupFallback(void* elem, void (*cleanup)(void*)); // cleanup function ptr.
void CleanupListFallback(); struct CleanupNode {
void* elem; // Pointer to the object to be cleaned up.
void (*cleanup)(void*); // Function pointer to the destructor or deleter.
};
// Cleanup uses a chunked linked list, to reduce pointer chasing.
struct CleanupChunk {
static size_t SizeOf(size_t i) {
return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1));
}
size_t size; // Total elements in the list.
CleanupChunk* next; // Next node in the list.
CleanupNode nodes[1]; // True length is |size|.
};
ArenaImpl* arena_; // Containing arena. ArenaImpl* arena_; // Containing arena.
void* owner_; // &ThreadCache of this thread; void* owner_; // &ThreadCache of this thread;
@ -248,64 +238,137 @@ class PROTOBUF_EXPORT ArenaImpl {
// Next CleanupList members to append to. These point inside cleanup_. // Next CleanupList members to append to. These point inside cleanup_.
CleanupNode* cleanup_ptr_; CleanupNode* cleanup_ptr_;
CleanupNode* cleanup_limit_; CleanupNode* cleanup_limit_;
};
// Blocks are variable length malloc-ed objects. The following structure void* AllocateAlignedFallback(size_t n);
// describes the common header for all blocks. void AddCleanupFallback(void* elem, void (*cleanup)(void*));
class PROTOBUF_EXPORT Block { void CleanupListFallback();
public: public:
Block(size_t size, Block* next); static constexpr size_t kBlockHeaderSize =
(sizeof(Block) + 7) & static_cast<size_t>(-8);
};
char* Pointer(size_t n) { // This class provides the core Arena memory allocation library. Different
GOOGLE_DCHECK(n <= size_); // implementations only need to implement the public interface below.
return reinterpret_cast<char*>(this) + n; // Arena is not a template type as that would only be useful if all protos
// in turn would be templates, which will/cannot happen. However separating
// the memory allocation part from the cruft of the API users expect we can
// use #ifdef the select the best implementation based on hardware / OS.
class PROTOBUF_EXPORT ArenaImpl {
public:
static const size_t kDefaultStartBlockSize = 256;
static const size_t kDefaultMaxBlockSize = 8192;
ArenaImpl() { Init(false); }
ArenaImpl(char* mem, size_t size) {
GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(mem) & 7, 0u);
Init(false);
// Ignore initial block if it is too small.
if (mem != nullptr && size >= kBlockHeaderSize + kSerialArenaSize) {
SetInitialBlock(new (mem) SerialArena::Block(size, nullptr, true, true));
}
} }
Block* next() const { return next_; } explicit ArenaImpl(const ArenaOptions& options);
size_t pos() const { return pos_; }
size_t size() const { return size_; } // Destructor deletes all owned heap allocated objects, and destructs objects
void set_pos(size_t pos) { pos_ = pos; } // that have non-trivial destructors, except for proto2 message objects whose
// destructors can be skipped. Also, frees all blocks except the initial block
// if it was passed in.
~ArenaImpl();
uint64 Reset();
uint64 SpaceAllocated() const;
uint64 SpaceUsed() const;
void* AllocateAligned(size_t n) {
SerialArena* arena;
if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
return arena->AllocateAligned(n);
} else {
return AllocateAlignedFallback(n);
}
}
// This function allocates n bytes if the common happy case is true and
// returns true. Otherwise does nothing and returns false. This strange
// semantics is necessary to allow callers to program functions that only
// have fallback function calls in tail position. This substantially improves
// code for the happy path.
PROTOBUF_ALWAYS_INLINE bool MaybeAllocateAligned(size_t n, void** out) {
SerialArena* a;
if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFromThreadCache(&a))) {
return a->MaybeAllocateAligned(n, out);
}
return false;
}
void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*));
// Add object pointer and cleanup function pointer to the list.
void AddCleanup(void* elem, void (*cleanup)(void*));
inline void RecordAlloc(const std::type_info* allocated_type,
size_t n) const {
if (PROTOBUF_PREDICT_FALSE(record_allocs())) {
options_->metrics_collector->OnAlloc(allocated_type, n);
}
}
std::pair<void*, size_t> NewBuffer(size_t last_size, size_t min_bytes);
private: private:
Block* next_; // Next block for this thread. // Pointer to a linked list of SerialArena.
size_t pos_; std::atomic<SerialArena*> threads_;
size_t size_; std::atomic<SerialArena*> hint_; // Fast thread-local block access
// data follows std::atomic<size_t> space_allocated_; // Total size of all allocated blocks.
};
struct ThreadCache { // Unique for each arena. Changes on Reset().
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // Least-significant-bit is 1 iff allocations should be recorded.
// If we are using the ThreadLocalStorage class to store the ThreadCache, uint64 lifecycle_id_;
// then the ThreadCache's default constructor has to be responsible for
// initializing it.
ThreadCache() : last_lifecycle_id_seen(-1), last_serial_arena(NULL) {}
#endif
// The ThreadCache is considered valid as long as this matches the struct Options {
// lifecycle_id of the arena being used. size_t start_block_size;
LifecycleId last_lifecycle_id_seen; size_t max_block_size;
SerialArena* last_serial_arena; void* (*block_alloc)(size_t);
void (*block_dealloc)(void*, size_t);
ArenaMetricsCollector* metrics_collector;
}; };
static std::atomic<LifecycleId> lifecycle_id_generator_;
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// Android ndk does not support __thread keyword so we use a custom thread
// local storage class we implemented.
// iOS also does not support the __thread keyword.
static ThreadCache& thread_cache();
#elif defined(PROTOBUF_USE_DLLS)
// Thread local variables cannot be exposed through DLL interface but we can
// wrap them in static functions.
static ThreadCache& thread_cache();
#else
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_;
static ThreadCache& thread_cache() { return thread_cache_; }
#endif
void Init(); Options* options_ = nullptr;
void* AllocateAlignedFallback(size_t n);
void* AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*));
void AddCleanupFallback(void* elem, void (*cleanup)(void*));
void Init(bool record_allocs);
void SetInitialBlock(
SerialArena::Block* block); // Can be called right after Init()
// Return true iff allocations should be recorded in a metrics collector.
inline bool record_allocs() const { return lifecycle_id_ & 1; }
// Invoke fn(b) for every Block* b.
template <typename Functor>
void PerBlock(Functor fn) {
// By omitting an Acquire barrier we ensure that any user code that doesn't
// properly synchronize Reset() or the destructor will throw a TSAN warning.
SerialArena* serial = threads_.load(std::memory_order_relaxed);
while (serial) {
// fn() may delete blocks and arenas, so fetch next pointers before fn();
SerialArena* cur = serial;
serial = serial->next();
for (auto* block = cur->head(); block != nullptr;) {
auto* b = block;
block = b->next();
fn(b);
}
}
}
// Free all blocks and return the total space used which is the sums of sizes
// of the all the allocated blocks.
uint64 FreeBlocks();
// Delete or Destruct all objects owned by the arena. // Delete or Destruct all objects owned by the arena.
void CleanupList(); void CleanupList();
@ -319,16 +382,6 @@ class PROTOBUF_EXPORT ArenaImpl {
hint_.store(serial, std::memory_order_release); hint_.store(serial, std::memory_order_release);
} }
std::atomic<SerialArena*>
threads_; // Pointer to a linked list of SerialArena.
std::atomic<SerialArena*> hint_; // Fast thread-local block access
std::atomic<size_t> space_allocated_; // Total size of all allocated blocks.
Block* initial_block_; // If non-NULL, points to the block that came from
// user data.
Block* NewBlock(Block* last_block, size_t min_bytes);
PROTOBUF_ALWAYS_INLINE bool GetSerialArenaFast(SerialArena** arena) { PROTOBUF_ALWAYS_INLINE bool GetSerialArenaFast(SerialArena** arena) {
if (GetSerialArenaFromThreadCache(arena)) return true; if (GetSerialArenaFromThreadCache(arena)) return true;
@ -356,9 +409,58 @@ class PROTOBUF_EXPORT ArenaImpl {
return false; return false;
} }
SerialArena* GetSerialArenaFallback(void* me); SerialArena* GetSerialArenaFallback(void* me);
LifecycleId lifecycle_id_; // Unique for each arena. Changes on Reset().
Options options_; #ifdef _MSC_VER
#pragma warning(disable : 4324)
#endif
struct alignas(64) ThreadCache {
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// If we are using the ThreadLocalStorage class to store the ThreadCache,
// then the ThreadCache's default constructor has to be responsible for
// initializing it.
ThreadCache()
: next_lifecycle_id(0),
last_lifecycle_id_seen(-1),
last_serial_arena(NULL) {}
#endif
// Number of per-thread lifecycle IDs to reserve. Must be power of two.
// To reduce contention on a global atomic, each thread reserves a batch of
// IDs. The following number is calculated based on a stress test with
// ~6500 threads all frequently allocating a new arena.
static constexpr size_t kPerThreadIds = 256;
// Next lifecycle ID available to this thread. We need to reserve a new
// batch, if `next_lifecycle_id & (kPerThreadIds - 1) == 0`.
uint64 next_lifecycle_id;
// The ThreadCache is considered valid as long as this matches the
// lifecycle_id of the arena being used.
uint64 last_lifecycle_id_seen;
SerialArena* last_serial_arena;
};
// Lifecycle_id can be highly contended variable in a situation of lots of
// arena creation. Make sure that other global variables are not sharing the
// cacheline.
#ifdef _MSC_VER
#pragma warning(disable : 4324)
#endif
struct alignas(64) CacheAlignedLifecycleIdGenerator {
std::atomic<LifecycleIdAtomic> id;
};
static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_;
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// Android ndk does not support __thread keyword so we use a custom thread
// local storage class we implemented.
// iOS also does not support the __thread keyword.
static ThreadCache& thread_cache();
#elif defined(PROTOBUF_USE_DLLS)
// Thread local variables cannot be exposed through DLL interface but we can
// wrap them in static functions.
static ThreadCache& thread_cache();
#else
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_;
static ThreadCache& thread_cache() { return thread_cache_; }
#endif
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl); GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl);
// All protos have pointers back to the arena hence Arena must have // All protos have pointers back to the arena hence Arena must have
@ -369,10 +471,11 @@ class PROTOBUF_EXPORT ArenaImpl {
public: public:
// kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8
// to protect the invariant that pos is always at a multiple of 8. // to protect the invariant that pos is always at a multiple of 8.
static const size_t kBlockHeaderSize = static constexpr size_t kBlockHeaderSize = SerialArena::kBlockHeaderSize;
(sizeof(Block) + 7) & static_cast<size_t>(-8); static constexpr size_t kSerialArenaSize =
static const size_t kSerialArenaSize =
(sizeof(SerialArena) + 7) & static_cast<size_t>(-8); (sizeof(SerialArena) + 7) & static_cast<size_t>(-8);
static constexpr size_t kOptionsSize =
(sizeof(Options) + 7) & static_cast<size_t>(-8);
static_assert(kBlockHeaderSize % 8 == 0, static_assert(kBlockHeaderSize % 8 == 0,
"kBlockHeaderSize must be a multiple of 8."); "kBlockHeaderSize must be a multiple of 8.");
static_assert(kSerialArenaSize % 8 == 0, static_assert(kSerialArenaSize % 8 == 0,

@ -31,6 +31,7 @@
#include <google/protobuf/arena.h> #include <google/protobuf/arena.h>
#include <algorithm> #include <algorithm>
#include <cstddef>
#include <cstring> #include <cstring>
#include <memory> #include <memory>
#include <string> #include <string>
@ -272,13 +273,20 @@ TEST(ArenaTest, CreateWithMoveArguments) {
} }
TEST(ArenaTest, InitialBlockTooSmall) { TEST(ArenaTest, InitialBlockTooSmall) {
// Construct a small (64 byte) initial block of memory to be used by the // Construct a small blocks of memory to be used by the arena allocator; then,
// arena allocator; then, allocate an object which will not fit in the // allocate an object which will not fit in the initial block.
// initial block. for (int size = 0; size <= Arena::kBlockOverhead + 32; size++) {
std::vector<char> arena_block(96); std::vector<char> arena_block(size);
ArenaOptions options; ArenaOptions options;
options.initial_block = &arena_block[0]; options.initial_block = arena_block.data();
options.initial_block_size = arena_block.size(); options.initial_block_size = arena_block.size();
// Try sometimes with non-default block sizes so that we exercise paths
// with and without ArenaImpl::Options.
if ((size % 2) != 0) {
options.start_block_size += 8;
}
Arena arena(options); Arena arena(options);
char* p = Arena::CreateArray<char>(&arena, 96); char* p = Arena::CreateArray<char>(&arena, 96);
@ -286,7 +294,7 @@ TEST(ArenaTest, InitialBlockTooSmall) {
// Ensure that the arena allocator did not return memory pointing into the // Ensure that the arena allocator did not return memory pointing into the
// initial block of memory. // initial block of memory.
uintptr_t arena_start = reinterpret_cast<uintptr_t>(&arena_block[0]); uintptr_t arena_start = reinterpret_cast<uintptr_t>(arena_block.data());
uintptr_t arena_end = arena_start + arena_block.size(); uintptr_t arena_end = arena_start + arena_block.size();
EXPECT_FALSE(allocation >= arena_start && allocation < arena_end); EXPECT_FALSE(allocation >= arena_start && allocation < arena_end);
@ -294,6 +302,7 @@ TEST(ArenaTest, InitialBlockTooSmall) {
// trigger a check for heap corruption if the object was allocated from the // trigger a check for heap corruption if the object was allocated from the
// initially-provided block. // initially-provided block.
memset(p, '\0', 96); memset(p, '\0', 96);
}
} }
TEST(ArenaTest, Parsing) { TEST(ArenaTest, Parsing) {
@ -988,10 +997,7 @@ TEST(ArenaTest, ExtensionsOnArena) {
TEST(ArenaTest, RepeatedFieldOnArena) { TEST(ArenaTest, RepeatedFieldOnArena) {
// Preallocate an initial arena block to avoid mallocs during hooked region. // Preallocate an initial arena block to avoid mallocs during hooked region.
std::vector<char> arena_block(1024 * 1024); std::vector<char> arena_block(1024 * 1024);
ArenaOptions options; Arena arena(arena_block.data(), arena_block.size());
options.initial_block = &arena_block[0];
options.initial_block_size = arena_block.size();
Arena arena(options);
{ {
internal::NoHeapChecker no_heap; internal::NoHeapChecker no_heap;
@ -1189,10 +1195,7 @@ TEST(ArenaTest, RepeatedFieldWithNonPODType) {
uint64 Align8(uint64 n) { return (n + 7) & -8; } uint64 Align8(uint64 n) { return (n + 7) & -8; }
TEST(ArenaTest, SpaceAllocated_and_Used) { TEST(ArenaTest, SpaceAllocated_and_Used) {
ArenaOptions options; Arena arena_1;
options.start_block_size = 256;
options.max_block_size = 8192;
Arena arena_1(options);
EXPECT_EQ(0, arena_1.SpaceAllocated()); EXPECT_EQ(0, arena_1.SpaceAllocated());
EXPECT_EQ(0, arena_1.SpaceUsed()); EXPECT_EQ(0, arena_1.SpaceUsed());
EXPECT_EQ(0, arena_1.Reset()); EXPECT_EQ(0, arena_1.Reset());
@ -1204,6 +1207,9 @@ TEST(ArenaTest, SpaceAllocated_and_Used) {
// Test with initial block. // Test with initial block.
std::vector<char> arena_block(1024); std::vector<char> arena_block(1024);
ArenaOptions options;
options.start_block_size = 256;
options.max_block_size = 8192;
options.initial_block = &arena_block[0]; options.initial_block = &arena_block[0];
options.initial_block_size = arena_block.size(); options.initial_block_size = arena_block.size();
Arena arena_2(options); Arena arena_2(options);
@ -1214,19 +1220,25 @@ TEST(ArenaTest, SpaceAllocated_and_Used) {
EXPECT_EQ(1024, arena_2.SpaceAllocated()); EXPECT_EQ(1024, arena_2.SpaceAllocated());
EXPECT_EQ(Align8(55), arena_2.SpaceUsed()); EXPECT_EQ(Align8(55), arena_2.SpaceUsed());
EXPECT_EQ(1024, arena_2.Reset()); EXPECT_EQ(1024, arena_2.Reset());
}
TEST(ArenaTest, BlockSizeDoubling) {
Arena arena;
EXPECT_EQ(0, arena.SpaceUsed());
EXPECT_EQ(0, arena.SpaceAllocated());
// Reset options to test doubling policy explicitly. // Allocate something to get initial block size.
options.initial_block = NULL; Arena::CreateArray<char>(&arena, 1);
options.initial_block_size = 0; auto first_block_size = arena.SpaceAllocated();
Arena arena_3(options);
EXPECT_EQ(0, arena_3.SpaceUsed()); // Keep allocating until space used increases.
Arena::CreateArray<char>(&arena_3, 160); while (arena.SpaceAllocated() == first_block_size) {
EXPECT_EQ(256, arena_3.SpaceAllocated()); Arena::CreateArray<char>(&arena, 1);
EXPECT_EQ(Align8(160), arena_3.SpaceUsed()); }
Arena::CreateArray<char>(&arena_3, 70); ASSERT_GT(arena.SpaceAllocated(), first_block_size);
EXPECT_EQ(256 + 512, arena_3.SpaceAllocated()); auto second_block_size = (arena.SpaceAllocated() - first_block_size);
EXPECT_EQ(Align8(160) + Align8(70), arena_3.SpaceUsed());
EXPECT_EQ(256 + 512, arena_3.Reset()); EXPECT_EQ(second_block_size, 2*first_block_size);
} }
TEST(ArenaTest, Alignment) { TEST(ArenaTest, Alignment) {
@ -1304,81 +1316,93 @@ TEST(ArenaTest, AddCleanup) {
} }
} }
// A helper utility class to only contain static hook functions, some namespace {
// counters to be used to verify the counters have been called and a cookie uint32 hooks_num_init = 0;
// value to be verified. uint32 hooks_num_allocations = 0;
class ArenaHooksTestUtil { uint32 hooks_num_reset = 0;
uint32 hooks_num_destruct = 0;
void ClearHookCounts() {
hooks_num_init = 0;
hooks_num_allocations = 0;
hooks_num_reset = 0;
hooks_num_destruct = 0;
}
} // namespace
// A helper utility class that handles arena callbacks.
class ArenaOptionsTestFriend : public internal::ArenaMetricsCollector {
public: public:
static void* on_init(Arena* arena) { static internal::ArenaMetricsCollector* NewWithAllocs() {
++num_init; return new ArenaOptionsTestFriend(true);
int* cookie = new int(kCookieValue);
return static_cast<void*>(cookie);
} }
static void on_allocation(const std::type_info* /*unused*/, uint64 alloc_size, static internal::ArenaMetricsCollector* NewWithoutAllocs() {
void* cookie) { return new ArenaOptionsTestFriend(false);
++num_allocations;
int cookie_value = *static_cast<int*>(cookie);
EXPECT_EQ(kCookieValue, cookie_value);
} }
static void on_reset(Arena* arena, void* cookie, uint64 space_used) { static void Enable(ArenaOptions* options) {
++num_reset; ClearHookCounts();
int cookie_value = *static_cast<int*>(cookie); options->make_metrics_collector = &ArenaOptionsTestFriend::NewWithAllocs;
EXPECT_EQ(kCookieValue, cookie_value);
} }
static void on_destruction(Arena* arena, void* cookie, uint64 space_used) { static void EnableWithoutAllocs(ArenaOptions* options) {
++num_destruct; ClearHookCounts();
int cookie_value = *static_cast<int*>(cookie); options->make_metrics_collector = &ArenaOptionsTestFriend::NewWithoutAllocs;
EXPECT_EQ(kCookieValue, cookie_value);
delete static_cast<int*>(cookie);
} }
static const int kCookieValue = 999; explicit ArenaOptionsTestFriend(bool record_allocs)
static uint32 num_init; : record_allocs_(record_allocs) {
static uint32 num_allocations; ++hooks_num_init;
static uint32 num_reset; }
static uint32 num_destruct; void OnDestroy(uint64 space_allocated) override {
}; ++hooks_num_destruct;
uint32 ArenaHooksTestUtil::num_init = 0; delete this;
uint32 ArenaHooksTestUtil::num_allocations = 0;
uint32 ArenaHooksTestUtil::num_reset = 0;
uint32 ArenaHooksTestUtil::num_destruct = 0;
const int ArenaHooksTestUtil::kCookieValue;
class ArenaOptionsTestFriend {
public:
static void Set(ArenaOptions* options) {
options->on_arena_init = ArenaHooksTestUtil::on_init;
options->on_arena_allocation = ArenaHooksTestUtil::on_allocation;
options->on_arena_reset = ArenaHooksTestUtil::on_reset;
options->on_arena_destruction = ArenaHooksTestUtil::on_destruction;
} }
void OnReset(uint64 space_allocated) override { ++hooks_num_reset; }
bool RecordAllocs() override { return record_allocs_; }
void OnAlloc(const std::type_info* allocated_type,
uint64 alloc_size) override {
++hooks_num_allocations;
}
private:
bool record_allocs_;
}; };
// Test the hooks are correctly called and that the cookie is passed. // Test the hooks are correctly called.
TEST(ArenaTest, ArenaHooksSanity) { TEST(ArenaTest, ArenaHooksSanity) {
ArenaOptions options; ArenaOptions options;
ArenaOptionsTestFriend::Set(&options); ArenaOptionsTestFriend::Enable(&options);
// Scope for defining the arena // Scope for defining the arena
{ {
Arena arena(options); Arena arena(options);
EXPECT_EQ(1, ArenaHooksTestUtil::num_init); EXPECT_EQ(1, hooks_num_init);
EXPECT_EQ(0, ArenaHooksTestUtil::num_allocations); EXPECT_EQ(0, hooks_num_allocations);
Arena::Create<uint64>(&arena); Arena::Create<uint64>(&arena);
if (std::is_trivially_destructible<uint64>::value) { if (std::is_trivially_destructible<uint64>::value) {
EXPECT_EQ(1, ArenaHooksTestUtil::num_allocations); EXPECT_EQ(1, hooks_num_allocations);
} else { } else {
EXPECT_EQ(2, ArenaHooksTestUtil::num_allocations); EXPECT_EQ(2, hooks_num_allocations);
} }
arena.Reset(); arena.Reset();
arena.Reset(); arena.Reset();
EXPECT_EQ(2, ArenaHooksTestUtil::num_reset); EXPECT_EQ(2, hooks_num_reset);
} }
EXPECT_EQ(3, ArenaHooksTestUtil::num_reset); EXPECT_EQ(2, hooks_num_reset);
EXPECT_EQ(1, ArenaHooksTestUtil::num_destruct); EXPECT_EQ(1, hooks_num_destruct);
}
// Test that allocation hooks are not called when we don't need them.
TEST(ArenaTest, ArenaHooksWhenAllocationsNotNeeded) {
ArenaOptions options;
ArenaOptionsTestFriend::EnableWithoutAllocs(&options);
Arena arena(options);
EXPECT_EQ(0, hooks_num_allocations);
Arena::Create<uint64>(&arena);
EXPECT_EQ(0, hooks_num_allocations);
} }

@ -0,0 +1,254 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <google/protobuf/arenastring.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/parse_context.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/message_lite.h>
#include <google/protobuf/stubs/mutex.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/stl_util.h>
// clang-format off
#include <google/protobuf/port_def.inc>
// clang-format on
namespace google {
namespace protobuf {
namespace internal {
const std::string& LazyString::Init() const {
static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED};
mu.Lock();
const std::string* res = inited_.load(std::memory_order_acquire);
if (res == nullptr) {
auto init_value = init_value_;
res = ::new (static_cast<void*>(string_buf_))
std::string(init_value.ptr, init_value.size);
inited_.store(res, std::memory_order_release);
}
mu.Unlock();
return *res;
}
void ArenaStringPtr::Set(const std::string* default_value,
ConstStringParam value, ::google::protobuf::Arena* arena) {
if (IsDefault(default_value)) {
tagged_ptr_.Set(Arena::Create<std::string>(arena, value));
} else {
UnsafeMutablePointer()->assign(value.data(), value.length());
}
}
void ArenaStringPtr::Set(const std::string* default_value, std::string&& value,
::google::protobuf::Arena* arena) {
if (IsDefault(default_value)) {
if (arena == nullptr) {
tagged_ptr_.Set(new std::string(std::move(value)));
} else {
tagged_ptr_.Set(Arena::Create<std::string>(arena, std::move(value)));
}
} else if (IsDonatedString()) {
std::string* current = tagged_ptr_.Get();
auto* s = new (current) std::string(std::move(value));
arena->OwnDestructor(s);
tagged_ptr_.Set(s);
} else /* !IsDonatedString() */ {
*UnsafeMutablePointer() = std::move(value);
}
}
void ArenaStringPtr::Set(EmptyDefault, ConstStringParam value,
::google::protobuf::Arena* arena) {
Set(&GetEmptyStringAlreadyInited(), value, arena);
}
void ArenaStringPtr::Set(EmptyDefault, std::string&& value,
::google::protobuf::Arena* arena) {
Set(&GetEmptyStringAlreadyInited(), std::move(value), arena);
}
void ArenaStringPtr::Set(NonEmptyDefault, ConstStringParam value,
::google::protobuf::Arena* arena) {
Set(nullptr, value, arena);
}
void ArenaStringPtr::Set(NonEmptyDefault, std::string&& value,
::google::protobuf::Arena* arena) {
Set(nullptr, std::move(value), arena);
}
std::string* ArenaStringPtr::Mutable(EmptyDefault, ::google::protobuf::Arena* arena) {
if (!IsDonatedString() && !IsDefault(&GetEmptyStringAlreadyInited())) {
return UnsafeMutablePointer();
} else {
return MutableSlow(arena);
}
}
std::string* ArenaStringPtr::Mutable(const LazyString& default_value,
::google::protobuf::Arena* arena) {
if (!IsDonatedString() && !IsDefault(nullptr)) {
return UnsafeMutablePointer();
} else {
return MutableSlow(arena, default_value);
}
}
std::string* ArenaStringPtr::MutableNoCopy(const std::string* default_value,
::google::protobuf::Arena* arena) {
if (!IsDonatedString() && !IsDefault(default_value)) {
return UnsafeMutablePointer();
} else {
GOOGLE_DCHECK(IsDefault(default_value));
// Allocate empty. The contents are not relevant.
std::string* new_string = Arena::Create<std::string>(arena);
tagged_ptr_.Set(new_string);
return new_string;
}
}
template <typename... Lazy>
std::string* ArenaStringPtr::MutableSlow(::google::protobuf::Arena* arena,
const Lazy&... lazy_default) {
const std::string* const default_value =
sizeof...(Lazy) == 0 ? &GetEmptyStringAlreadyInited() : nullptr;
GOOGLE_DCHECK(IsDefault(default_value));
std::string* new_string =
Arena::Create<std::string>(arena, lazy_default.get()...);
tagged_ptr_.Set(new_string);
return new_string;
}
std::string* ArenaStringPtr::Release(const std::string* default_value,
::google::protobuf::Arena* arena) {
if (IsDefault(default_value)) {
return nullptr;
} else {
return ReleaseNonDefault(default_value, arena);
}
}
std::string* ArenaStringPtr::ReleaseNonDefault(const std::string* default_value,
::google::protobuf::Arena* arena) {
GOOGLE_DCHECK(!IsDefault(default_value));
if (!IsDonatedString()) {
std::string* released;
if (arena != nullptr) {
released = new std::string;
released->swap(*UnsafeMutablePointer());
} else {
released = UnsafeMutablePointer();
}
tagged_ptr_.Set(const_cast<std::string*>(default_value));
return released;
} else /* IsDonatedString() */ {
GOOGLE_DCHECK(arena != nullptr);
std::string* released = new std::string(Get());
tagged_ptr_.Set(const_cast<std::string*>(default_value));
return released;
}
}
void ArenaStringPtr::SetAllocated(const std::string* default_value,
std::string* value, ::google::protobuf::Arena* arena) {
// Release what we have first.
if (arena == nullptr && !IsDefault(default_value)) {
delete UnsafeMutablePointer();
}
if (value == nullptr) {
tagged_ptr_.Set(const_cast<std::string*>(default_value));
} else {
#ifdef NDEBUG
tagged_ptr_.Set(value);
if (arena != nullptr) {
arena->Own(value);
}
#else
// On debug builds, copy the string so the address differs. delete will
// fail if value was a stack-allocated temporary/etc., which would have
// failed when arena ran its cleanup list.
std::string* new_value = Arena::Create<std::string>(arena, *value);
delete value;
tagged_ptr_.Set(new_value);
#endif
}
}
void ArenaStringPtr::Destroy(const std::string* default_value,
::google::protobuf::Arena* arena) {
if (arena == nullptr) {
GOOGLE_DCHECK(!IsDonatedString());
if (!IsDefault(default_value)) {
delete UnsafeMutablePointer();
}
}
}
void ArenaStringPtr::Destroy(EmptyDefault, ::google::protobuf::Arena* arena) {
Destroy(&GetEmptyStringAlreadyInited(), arena);
}
void ArenaStringPtr::Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena) {
Destroy(nullptr, arena);
}
void ArenaStringPtr::ClearToEmpty() {
if (IsDefault(&GetEmptyStringAlreadyInited())) {
// Already set to default -- do nothing.
} else {
// Unconditionally mask away the tag.
//
// UpdateDonatedString uses assign when capacity is larger than the new
// value, which is trivially true in the donated string case.
// const_cast<std::string*>(PtrValue<std::string>())->clear();
tagged_ptr_.Get()->clear();
}
}
void ArenaStringPtr::ClearToDefault(const LazyString& default_value,
::google::protobuf::Arena* arena) {
(void)arena;
if (IsDefault(nullptr)) {
// Already set to default -- do nothing.
} else if (!IsDonatedString()) {
UnsafeMutablePointer()->assign(default_value.get());
}
}
} // namespace internal
} // namespace protobuf
} // namespace google

@ -37,7 +37,6 @@
#include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/fastmem.h>
#include <google/protobuf/arena.h> #include <google/protobuf/arena.h>
#include <google/protobuf/port.h> #include <google/protobuf/port.h>
@ -48,261 +47,329 @@
#endif #endif
// This is the implementation of arena string fields written for the open-source
// release. The ArenaStringPtr struct below is an internal implementation class
// and *should not be used* by user code. It is used to collect string
// operations together into one place and abstract away the underlying
// string-field pointer representation, so that (for example) an alternate
// implementation that knew more about ::std::string's internals could integrate
// more closely with the arena allocator.
namespace google { namespace google {
namespace protobuf { namespace protobuf {
namespace internal { namespace internal {
// Lazy string instance to support string fields with non-empty default.
// These are initialized on the first call to .get().
class PROTOBUF_EXPORT LazyString {
public:
// We explicitly make LazyString an aggregate so that MSVC can do constant
// initialization on it without marking it `constexpr`.
// We do not want to use `constexpr` because it makes it harder to have extern
// storage for it and causes library bloat.
struct InitValue {
const char* ptr;
size_t size;
};
// We keep a union of the initialization value and the std::string to save on
// space. We don't need the string array after Init() is done.
union {
mutable InitValue init_value_;
alignas(std::string) mutable char string_buf_[sizeof(std::string)];
};
mutable std::atomic<const std::string*> inited_;
const std::string& get() const {
// This check generates less code than a call-once invocation.
auto* res = inited_.load(std::memory_order_acquire);
if (PROTOBUF_PREDICT_FALSE(res == nullptr)) return Init();
return *res;
}
private:
// Initialize the string in `string_buf_`, update `inited_` and return it.
// We return it here to avoid having to read it again in the inlined code.
const std::string& Init() const;
};
template <typename T> template <typename T>
class TaggedPtr { class TaggedPtr {
public: public:
void Set(T* p) { ptr_ = reinterpret_cast<uintptr_t>(p); } TaggedPtr() = default;
T* Get() const { return reinterpret_cast<T*>(ptr_); } explicit constexpr TaggedPtr(const std::string* ptr)
: ptr_(const_cast<std::string*>(ptr)) {}
bool IsNull() { return ptr_ == 0; } void SetTagged(T* p) {
Set(p);
ptr_ = reinterpret_cast<void*>(as_int() | 1);
}
void Set(T* p) { ptr_ = p; }
T* Get() const { return reinterpret_cast<T*>(as_int() & -2); }
bool IsTagged() const { return as_int() & 1; }
// Returned value is only safe to dereference if IsTagged() == false.
// It is safe to compare.
T* UnsafeGet() const { return static_cast<T*>(ptr_); }
bool IsNull() { return ptr_ == nullptr; }
private: private:
uintptr_t ptr_; uintptr_t as_int() const { return reinterpret_cast<uintptr_t>(ptr_); }
void* ptr_;
}; };
struct PROTOBUF_EXPORT ArenaStringPtr { static_assert(std::is_trivial<TaggedPtr<std::string>>::value,
inline void Set(const ::std::string* default_value, "TaggedPtr must be trivial");
const ::std::string& value, Arena* arena) {
if (ptr_ == default_value) {
CreateInstance(arena, &value);
} else {
*ptr_ = value;
}
}
inline void SetLite(const ::std::string* default_value, // This class encapsulates a pointer to a std::string with or without a donated
const ::std::string& value, Arena* arena) { // buffer, tagged by bottom bit. It is a high-level wrapper that almost directly
Set(default_value, value, arena); // corresponds to the interface required by string fields in generated
} // code. It replaces the old std::string* pointer in such cases.
//
// The object has different but similar code paths for when the default value is
// the empty string and when it is a non-empty string.
// The empty string is handled different throughout the library and there is a
// single global instance of it we can share.
//
// For fields with an empty string default value, there are three distinct
// states:
//
// - Pointer set to 'String' tag (LSB is 0), equal to
// &GetEmptyStringAlreadyInited(): field is set to its default value. Points
// to a true std::string*, but we do not own that std::string* (it's a
// globally shared instance).
//
// - Pointer set to 'String' tag (LSB is 0), but not equal to the global empty
// string: field points to a true std::string* instance that we own. This
// instance is either on the heap or on the arena (i.e. registered on
// free()/destructor-call list) as appropriate.
//
// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string
// instance with a buffer on the arena (arena != NULL, always, in this case).
//
// For fields with a non-empty string default value, there are three distinct
// states:
//
// - Pointer set to 'String' tag (LSB is 0), equal to `nullptr`:
// Field is in "default" mode and does not point to any actual instance.
// Methods that might need to create an instance of the object will pass a
// `const LazyString&` for it.
//
// - Pointer set to 'String' tag (LSB is 0), but not equal to `nullptr`:
// field points to a true std::string* instance that we own. This instance is
// either on the heap or on the arena (i.e. registered on
// free()/destructor-call list) as appropriate.
//
// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string
// instance with a buffer on the arena (arena != NULL, always, in this case).
//
// Generated code and reflection code both ensure that ptr_ is never null for
// fields with an empty default.
// Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and
// so the field is always manually initialized via method calls.
//
// Side-note: why pass information about the default on every API call? Because
// we don't want to hold it in a member variable, or else this would go into
// every proto message instance. This would be a huge waste of space, since the
// default instance pointer is typically a global (static class field). We want
// the generated code to be as efficient as possible, and if we take
// the default value information as a parameter that's in practice taken from a
// static class field, and compare ptr_ to the default value, we end up with a
// single "cmp %reg, GLOBAL" in the resulting machine code. (Note that this also
// requires the String tag to be 0 so we can avoid the mask before comparing.)
struct PROTOBUF_EXPORT ArenaStringPtr {
ArenaStringPtr() = default;
explicit constexpr ArenaStringPtr(const std::string* default_value)
: tagged_ptr_(default_value) {}
// Some methods below are overloaded on a `default_value` and on tags.
// The tagged overloads help reduce code size in the callers in generated
// code, while the `default_value` overloads are useful from reflection.
// By-value empty struct arguments are elided in the ABI.
struct EmptyDefault {};
struct NonEmptyDefault {};
void Set(const std::string* default_value, ConstStringParam value,
::google::protobuf::Arena* arena);
void Set(const std::string* default_value, std::string&& value,
::google::protobuf::Arena* arena);
void Set(EmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena);
void Set(EmptyDefault, std::string&& value, ::google::protobuf::Arena* arena);
void Set(NonEmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena);
void Set(NonEmptyDefault, std::string&& value, ::google::protobuf::Arena* arena);
// Basic accessors. // Basic accessors.
inline const ::std::string& Get() const { return *ptr_; } const std::string& Get() const PROTOBUF_ALWAYS_INLINE {
// Unconditionally mask away the tag.
return *tagged_ptr_.Get();
}
const std::string* GetPointer() const PROTOBUF_ALWAYS_INLINE {
// Unconditionally mask away the tag.
return tagged_ptr_.Get();
}
// For fields with an empty default value.
std::string* Mutable(EmptyDefault, ::google::protobuf::Arena* arena);
// For fields with a non-empty default value.
std::string* Mutable(const LazyString& default_value, ::google::protobuf::Arena* arena);
// Release returns a std::string* instance that is heap-allocated and is not
// Own()'d by any arena. If the field is not set, this returns NULL. The
// caller retains ownership. Clears this field back to NULL state. Used to
// implement release_<field>() methods on generated classes.
std::string* Release(const std::string* default_value,
::google::protobuf::Arena* arena);
std::string* ReleaseNonDefault(const std::string* default_value,
::google::protobuf::Arena* arena);
// Takes a std::string that is heap-allocated, and takes ownership. The
// std::string's destructor is registered with the arena. Used to implement
// set_allocated_<field> in generated classes.
void SetAllocated(const std::string* default_value, std::string* value,
::google::protobuf::Arena* arena);
inline ::std::string* Mutable(const ::std::string* default_value, // Swaps internal pointers. Arena-safety semantics: this is guarded by the
Arena* arena) { // logic in Swap()/UnsafeArenaSwap() at the message level, so this method is
if (ptr_ == default_value) { // 'unsafe' if called directly.
CreateInstance(arena, default_value); inline void Swap(ArenaStringPtr* other, const std::string* default_value,
} Arena* arena) PROTOBUF_ALWAYS_INLINE;
return ptr_;
}
// Release returns a ::std::string* instance that is heap-allocated and is not // Frees storage (if not on an arena).
// Own()'d by any arena. If the field was not set, it returns NULL. The caller void Destroy(const std::string* default_value, ::google::protobuf::Arena* arena);
// retains ownership. Clears this field back to NULL state. Used to implement void Destroy(EmptyDefault, ::google::protobuf::Arena* arena);
// release_<field>() methods on generated classes. void Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena);
inline ::std::string* Release(const ::std::string* default_value,
Arena* arena) {
if (ptr_ == default_value) {
return NULL;
}
return ReleaseNonDefault(default_value, arena);
}
// Similar to Release, but ptr_ cannot be the default_value. // Clears content, but keeps allocated std::string, to avoid the overhead of
inline ::std::string* ReleaseNonDefault(const ::std::string* default_value, // heap operations. After this returns, the content (as seen by the user) will
Arena* arena) { // always be the empty std::string. Assumes that |default_value| is an empty
GOOGLE_DCHECK(!IsDefault(default_value)); // std::string.
::std::string* released = NULL; void ClearToEmpty();
if (arena != NULL) {
// ptr_ is owned by the arena.
released = new ::std::string;
released->swap(*ptr_);
} else {
released = ptr_;
}
ptr_ = const_cast< ::std::string*>(default_value);
return released;
}
// UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned // Clears content, assuming that the current value is not the empty
// (i.e. have its destructor already registered) if arena != NULL. If the // string default.
// field was not set, this returns NULL. This method clears this field back to void ClearNonDefaultToEmpty();
// NULL state. Used to implement unsafe_arena_release_<field>() methods on
// generated classes.
inline ::std::string* UnsafeArenaRelease(const ::std::string* default_value,
Arena* /* arena */) {
if (ptr_ == default_value) {
return NULL;
}
::std::string* released = ptr_;
ptr_ = const_cast< ::std::string*>(default_value);
return released;
}
// Takes a string that is heap-allocated, and takes ownership. The string's // Clears content, but keeps allocated std::string if arena != NULL, to avoid
// destructor is registered with the arena. Used to implement // the overhead of heap operations. After this returns, the content (as seen
// set_allocated_<field> in generated classes. // by the user) will always be equal to |default_value|.
inline void SetAllocated(const ::std::string* default_value, void ClearToDefault(const LazyString& default_value, ::google::protobuf::Arena* arena);
::std::string* value, Arena* arena) {
if (arena == NULL && ptr_ != default_value) {
Destroy(default_value, arena);
}
if (value != NULL) {
ptr_ = value;
if (arena != NULL) {
arena->Own(value);
}
} else {
ptr_ = const_cast< ::std::string*>(default_value);
}
}
// Takes a string that has lifetime equal to the arena's lifetime. The arena // Called from generated code / reflection runtime only. Resets value to point
// must be non-null. It is safe only to pass this method a value returned by // to a default string pointer, with the semantics that this
// UnsafeArenaRelease() on another field of a message in the same arena. Used // ArenaStringPtr does not own the pointed-to memory. Disregards initial value
// to implement unsafe_arena_set_allocated_<field> in generated classes. // of ptr_ (so this is the *ONLY* safe method to call after construction or
inline void UnsafeArenaSetAllocated(const ::std::string* default_value, // when reinitializing after becoming the active field in a oneof union).
::std::string* value, inline void UnsafeSetDefault(const std::string* default_value);
Arena* /* arena */) {
if (value != NULL) { // Returns a mutable pointer, but doesn't initialize the string to the
ptr_ = value; // default value.
} else { std::string* MutableNoArenaNoDefault(const std::string* default_value);
ptr_ = const_cast< ::std::string*>(default_value);
} // Get a mutable pointer with unspecified contents.
// Similar to `MutableNoArenaNoDefault`, but also handles the arena case.
// If the value was donated, the contents are discarded.
std::string* MutableNoCopy(const std::string* default_value,
::google::protobuf::Arena* arena);
// Destroy the string. Assumes `arena == nullptr`.
void DestroyNoArena(const std::string* default_value);
// Internal setter used only at parse time to directly set a donated string
// value.
void UnsafeSetTaggedPointer(TaggedPtr<std::string> value) {
tagged_ptr_ = value;
} }
// Generated code only! An optimization, in certain cases the generated
// code is certain we can obtain a std::string with no default checks and
// tag tests.
std::string* UnsafeMutablePointer() PROTOBUF_RETURNS_NONNULL;
// Swaps internal pointers. Arena-safety semantics: this is guarded by the inline bool IsDefault(const std::string* default_value) const {
// logic in Swap()/UnsafeArenaSwap() at the message level, so this method is // Relies on the fact that kPtrTagString == 0, so if IsString(), ptr_ is the
// 'unsafe' if called directly. // actual std::string pointer (and if !IsString(), ptr_ will never be equal
PROTOBUF_ALWAYS_INLINE void Swap(ArenaStringPtr* other) { // to any aligned |default_value| pointer). The key is that we want to avoid
std::swap(ptr_, other->ptr_); // masking in the fastpath const-pointer Get() case for non-arena code.
return tagged_ptr_.UnsafeGet() == default_value;
} }
PROTOBUF_ALWAYS_INLINE void Swap(ArenaStringPtr* other,
const ::std::string* default_value, private:
TaggedPtr<std::string> tagged_ptr_;
bool IsDonatedString() const { return false; }
// Slow paths.
// MutableSlow requires that !IsString() || IsDefault
// Variadic to support 0 args for EmptyDefault and 1 arg for LazyString.
template <typename... Lazy>
std::string* MutableSlow(::google::protobuf::Arena* arena, const Lazy&... lazy_default);
};
inline void ArenaStringPtr::UnsafeSetDefault(const std::string* value) {
tagged_ptr_.Set(const_cast<std::string*>(value));
}
inline void ArenaStringPtr::Swap(ArenaStringPtr* other,
const std::string* default_value,
Arena* arena) { Arena* arena) {
#ifndef NDEBUG #ifndef NDEBUG
// For debug builds, we swap the contents of the string, rather than the // For debug builds, we swap the contents of the string, rather than the
// string instances themselves. This invalidates previously taken const // std::string instances themselves. This invalidates previously taken const
// references that are (per our documentation) invalidated by calling Swap() // references that are (per our documentation) invalidated by calling Swap()
// on the message. // on the message.
// //
// If both strings are the default_value, swapping is uninteresting. // If both strings are the default_value, swapping is uninteresting.
// Otherwise, we use ArenaStringPtr::Mutable() to access the string, to // Otherwise, we use ArenaStringPtr::Mutable() to access the std::string, to
// ensure that we do not try to mutate default_value itself. // ensure that we do not try to mutate default_value itself.
if (IsDefault(default_value) && other->IsDefault(default_value)) { if (IsDefault(default_value) && other->IsDefault(default_value)) {
return; return;
} }
::std::string* this_ptr = Mutable(default_value, arena); if (default_value == nullptr) {
::std::string* other_ptr = other->Mutable(default_value, arena); // If we have non-empty default, then `default_value` is null and we can't
// call Mutable the same way. Just do the regular swap.
std::swap(tagged_ptr_, other->tagged_ptr_);
} else {
std::string* this_ptr = Mutable(EmptyDefault{}, arena);
std::string* other_ptr = other->Mutable(EmptyDefault{}, arena);
this_ptr->swap(*other_ptr); this_ptr->swap(*other_ptr);
}
#else #else
std::swap(ptr_, other->ptr_); std::swap(tagged_ptr_, other->tagged_ptr_);
(void)default_value;
(void)arena;
#endif #endif
} }
// Frees storage (if not on an arena). inline void ArenaStringPtr::ClearNonDefaultToEmpty() {
inline void Destroy(const ::std::string* default_value, Arena* arena) { // Unconditionally mask away the tag.
if (arena == NULL && ptr_ != default_value) { tagged_ptr_.Get()->clear();
delete ptr_; }
}
} inline std::string* ArenaStringPtr::MutableNoArenaNoDefault(
const std::string* default_value) {
// Clears content, but keeps allocated string if arena != NULL, to avoid the // VERY IMPORTANT for performance and code size: this will reduce to a member
// overhead of heap operations. After this returns, the content (as seen by // variable load, a pointer check (against |default_value|, in practice a
// the user) will always be the empty string. Assumes that |default_value| // static global) and a branch to the slowpath (which calls operator new and
// is an empty string. // the ctor). DO NOT add any tagged-pointer operations here.
inline void ClearToEmpty(const ::std::string* default_value, if (IsDefault(default_value)) {
Arena* /* arena */) { std::string* new_string = new std::string();
if (ptr_ == default_value) { tagged_ptr_.Set(new_string);
// Already set to default (which is empty) -- do nothing. return new_string;
} else { } else {
ptr_->clear(); return UnsafeMutablePointer();
}
} }
}
// Clears content, assuming that the current value is not the empty string inline void ArenaStringPtr::DestroyNoArena(const std::string* default_value) {
// default. if (!IsDefault(default_value)) {
inline void ClearNonDefaultToEmpty() { ptr_->clear(); } delete UnsafeMutablePointer();
// Clears content, but keeps allocated string if arena != NULL, to avoid the
// overhead of heap operations. After this returns, the content (as seen by
// the user) will always be equal to |default_value|.
inline void ClearToDefault(const ::std::string* default_value,
Arena* /* arena */) {
if (ptr_ == default_value) {
// Already set to default -- do nothing.
} else {
// Have another allocated string -- rather than throwing this away and
// resetting ptr_ to the canonical default string instance, we just reuse
// this instance.
*ptr_ = *default_value;
}
} }
}
// Called from generated code / reflection runtime only. Resets value to point inline std::string* ArenaStringPtr::UnsafeMutablePointer() {
// to a default string pointer, with the semantics that this ArenaStringPtr GOOGLE_DCHECK(!tagged_ptr_.IsTagged());
// does not own the pointed-to memory. Disregards initial value of ptr_ (so GOOGLE_DCHECK(tagged_ptr_.UnsafeGet() != nullptr);
// this is the *ONLY* safe method to call after construction or when return tagged_ptr_.UnsafeGet();
// reinitializing after becoming the active field in a oneof union). }
inline void UnsafeSetDefault(const ::std::string* default_value) {
// Casting away 'const' is safe here: accessors ensure that ptr_ is only
// returned as a const if it is equal to default_value.
ptr_ = const_cast< ::std::string*>(default_value);
}
// Destroy the string. Assumes `arena == nullptr`.
inline void DestroyNoArena(const ::std::string* default_value) {
if (ptr_ != default_value) {
delete ptr_;
}
}
// Internal accessor used only at parse time to provide direct access to the
// raw pointer from the shared parse routine (in the non-arenas case). The
// parse routine does the string allocation in order to save code size in the
// generated parsing code.
inline ::std::string** UnsafeRawStringPointer() { return &ptr_; }
inline bool IsDefault(const ::std::string* default_value) const {
return ptr_ == default_value;
}
// Internal accessors!!!!
void UnsafeSetTaggedPointer(TaggedPtr< ::std::string> value) {
ptr_ = value.Get();
}
// Generated code only! An optimization, in certain cases the generated
// code is certain we can obtain a string with no default checks and
// tag tests.
::std::string* UnsafeMutablePointer() { return ptr_; }
private:
::std::string* ptr_;
PROTOBUF_NOINLINE
void CreateInstance(Arena* arena, const ::std::string* initial_value) {
GOOGLE_DCHECK(initial_value != NULL);
// uses "new ::std::string" when arena is nullptr
ptr_ = Arena::Create< ::std::string>(arena, *initial_value);
}
PROTOBUF_NOINLINE
void CreateInstanceNoArena(const ::std::string* initial_value) {
GOOGLE_DCHECK(initial_value != NULL);
ptr_ = new ::std::string(*initial_value);
}
};
} // namespace internal } // namespace internal
} // namespace protobuf } // namespace protobuf
} // namespace google } // namespace google
#include <google/protobuf/port_undef.inc> #include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_ARENASTRING_H__ #endif // GOOGLE_PROTOBUF_ARENASTRING_H__

@ -28,8 +28,6 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Based on mvels@'s frankenstring.
#include <google/protobuf/arenastring.h> #include <google/protobuf/arenastring.h>
#include <algorithm> #include <algorithm>
@ -42,10 +40,14 @@
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/coded_stream.h> #include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/generated_message_util.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <google/protobuf/stubs/strutil.h> #include <google/protobuf/stubs/strutil.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google { namespace google {
namespace protobuf { namespace protobuf {
@ -53,82 +55,119 @@ using internal::ArenaStringPtr;
static std::string WrapString(const char* value) { return value; } static std::string WrapString(const char* value) { return value; }
using EmptyDefault = ArenaStringPtr::EmptyDefault;
const internal::LazyString nonempty_default{{{"default", 7}}, {nullptr}};
// Test ArenaStringPtr with arena == NULL. // Test ArenaStringPtr with arena == NULL.
TEST(ArenaStringPtrTest, ArenaStringPtrOnHeap) { TEST(ArenaStringPtrTest, ArenaStringPtrOnHeap) {
ArenaStringPtr field; ArenaStringPtr field;
std::string default_value = "default"; const std::string* empty_default = &internal::GetEmptyString();
field.UnsafeSetDefault(&default_value); field.UnsafeSetDefault(empty_default);
EXPECT_EQ(std::string("default"), field.Get()); EXPECT_EQ(std::string(""), field.Get());
field.Set(&default_value, WrapString("Test short"), NULL); field.Set(empty_default, WrapString("Test short"), NULL);
EXPECT_EQ(std::string("Test short"), field.Get()); EXPECT_EQ(std::string("Test short"), field.Get());
field.Set(&default_value, WrapString("Test long long long long value"), NULL); field.Set(empty_default, WrapString("Test long long long long value"), NULL);
EXPECT_EQ(std::string("Test long long long long value"), field.Get()); EXPECT_EQ(std::string("Test long long long long value"), field.Get());
field.Set(&default_value, std::string(""), NULL); field.Set(empty_default, std::string(""), NULL);
field.Destroy(&default_value, NULL); field.Destroy(empty_default, NULL);
ArenaStringPtr field2; ArenaStringPtr field2;
field2.UnsafeSetDefault(&default_value); field2.UnsafeSetDefault(empty_default);
std::string* mut = field2.Mutable(&default_value, NULL); std::string* mut = field2.Mutable(EmptyDefault{}, NULL);
EXPECT_EQ(mut, field2.Mutable(&default_value, NULL)); EXPECT_EQ(mut, field2.Mutable(EmptyDefault{}, NULL));
EXPECT_EQ(mut, &field2.Get()); EXPECT_EQ(mut, &field2.Get());
EXPECT_NE(&default_value, mut); EXPECT_NE(empty_default, mut);
EXPECT_EQ(std::string("default"), *mut); EXPECT_EQ(std::string(""), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage *mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field2.Get()); EXPECT_EQ(std::string("Test long long long long value"), field2.Get());
field2.Destroy(&default_value, NULL); field2.Destroy(empty_default, NULL);
ArenaStringPtr field3;
field3.UnsafeSetDefault(nullptr);
mut = field3.Mutable(nonempty_default, NULL);
EXPECT_EQ(mut, field3.Mutable(nonempty_default, NULL));
EXPECT_EQ(mut, &field3.Get());
EXPECT_NE(nullptr, mut);
EXPECT_EQ(std::string("default"), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field3.Get());
field3.Destroy(nullptr, NULL);
} }
TEST(ArenaStringPtrTest, ArenaStringPtrOnArena) { TEST(ArenaStringPtrTest, ArenaStringPtrOnArena) {
Arena arena; Arena arena;
ArenaStringPtr field; ArenaStringPtr field;
std::string default_value = "default"; const std::string* empty_default = &internal::GetEmptyString();
field.UnsafeSetDefault(&default_value); field.UnsafeSetDefault(empty_default);
EXPECT_EQ(std::string("default"), field.Get()); EXPECT_EQ(std::string(""), field.Get());
field.Set(&default_value, WrapString("Test short"), &arena); field.Set(empty_default, WrapString("Test short"), &arena);
EXPECT_EQ(std::string("Test short"), field.Get()); EXPECT_EQ(std::string("Test short"), field.Get());
field.Set(&default_value, WrapString("Test long long long long value"), field.Set(empty_default, WrapString("Test long long long long value"),
&arena); &arena);
EXPECT_EQ(std::string("Test long long long long value"), field.Get()); EXPECT_EQ(std::string("Test long long long long value"), field.Get());
field.Set(&default_value, std::string(""), &arena); field.Set(empty_default, std::string(""), &arena);
field.Destroy(&default_value, &arena); field.Destroy(empty_default, &arena);
ArenaStringPtr field2; ArenaStringPtr field2;
field2.UnsafeSetDefault(&default_value); field2.UnsafeSetDefault(empty_default);
std::string* mut = field2.Mutable(&default_value, &arena); std::string* mut = field2.Mutable(EmptyDefault{}, &arena);
EXPECT_EQ(mut, field2.Mutable(&default_value, &arena)); EXPECT_EQ(mut, field2.Mutable(EmptyDefault{}, &arena));
EXPECT_EQ(mut, &field2.Get()); EXPECT_EQ(mut, &field2.Get());
EXPECT_NE(&default_value, mut); EXPECT_NE(empty_default, mut);
EXPECT_EQ(std::string("default"), *mut); EXPECT_EQ(std::string(""), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage *mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field2.Get()); EXPECT_EQ(std::string("Test long long long long value"), field2.Get());
field2.Destroy(&default_value, &arena); field2.Destroy(empty_default, &arena);
ArenaStringPtr field3;
field3.UnsafeSetDefault(nullptr);
mut = field3.Mutable(nonempty_default, &arena);
EXPECT_EQ(mut, field3.Mutable(nonempty_default, &arena));
EXPECT_EQ(mut, &field3.Get());
EXPECT_NE(nullptr, mut);
EXPECT_EQ(std::string("default"), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field3.Get());
field3.Destroy(nullptr, &arena);
} }
TEST(ArenaStringPtrTest, ArenaStringPtrOnArenaNoSSO) { TEST(ArenaStringPtrTest, ArenaStringPtrOnArenaNoSSO) {
Arena arena; Arena arena;
ArenaStringPtr field; ArenaStringPtr field;
std::string default_value = "default"; const std::string* empty_default = &internal::GetEmptyString();
field.UnsafeSetDefault(&default_value); field.UnsafeSetDefault(empty_default);
EXPECT_EQ(std::string("default"), field.Get()); EXPECT_EQ(std::string(""), field.Get());
// Avoid triggering the SSO optimization by setting the string to something // Avoid triggering the SSO optimization by setting the string to something
// larger than the internal buffer. // larger than the internal buffer.
field.Set(&default_value, WrapString("Test long long long long value"), field.Set(empty_default, WrapString("Test long long long long value"),
&arena); &arena);
EXPECT_EQ(std::string("Test long long long long value"), field.Get()); EXPECT_EQ(std::string("Test long long long long value"), field.Get());
field.Set(&default_value, std::string(""), &arena); field.Set(empty_default, std::string(""), &arena);
field.Destroy(&default_value, &arena); field.Destroy(empty_default, &arena);
ArenaStringPtr field2; ArenaStringPtr field2;
field2.UnsafeSetDefault(&default_value); field2.UnsafeSetDefault(empty_default);
std::string* mut = field2.Mutable(&default_value, &arena); std::string* mut = field2.Mutable(EmptyDefault{}, &arena);
EXPECT_EQ(mut, field2.Mutable(&default_value, &arena)); EXPECT_EQ(mut, field2.Mutable(EmptyDefault{}, &arena));
EXPECT_EQ(mut, &field2.Get()); EXPECT_EQ(mut, &field2.Get());
EXPECT_NE(&default_value, mut); EXPECT_NE(empty_default, mut);
EXPECT_EQ(std::string("default"), *mut); EXPECT_EQ(std::string(""), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage *mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field2.Get()); EXPECT_EQ(std::string("Test long long long long value"), field2.Get());
field2.Destroy(&default_value, &arena); field2.Destroy(empty_default, &arena);
ArenaStringPtr field3;
field3.UnsafeSetDefault(nullptr);
mut = field3.Mutable(nonempty_default, &arena);
EXPECT_EQ(mut, field3.Mutable(nonempty_default, &arena));
EXPECT_EQ(mut, &field3.Get());
EXPECT_NE(nullptr, mut);
EXPECT_EQ(std::string("default"), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field3.Get());
field3.Destroy(nullptr, &arena);
} }

@ -1233,6 +1233,7 @@ bool CommandLineInterface::AllowProto3Optional(
return false; return false;
} }
bool CommandLineInterface::VerifyInputFilesInDescriptors( bool CommandLineInterface::VerifyInputFilesInDescriptors(
DescriptorDatabase* database) { DescriptorDatabase* database) {
for (const auto& input_file : input_files_) { for (const auto& input_file : input_files_) {
@ -1251,6 +1252,7 @@ bool CommandLineInterface::VerifyInputFilesInDescriptors(
<< std::endl; << std::endl;
return false; return false;
} }
} }
return true; return true;
} }
@ -1301,6 +1303,7 @@ bool CommandLineInterface::ParseInputFiles(
break; break;
} }
// Enforce --direct_dependencies // Enforce --direct_dependencies
if (direct_dependencies_explicitly_set_) { if (direct_dependencies_explicitly_set_) {
bool indirect_imports = false; bool indirect_imports = false;
@ -1347,6 +1350,7 @@ void CommandLineInterface::Clear() {
disallow_services_ = false; disallow_services_ = false;
direct_dependencies_explicitly_set_ = false; direct_dependencies_explicitly_set_ = false;
allow_proto3_optional_ = false; allow_proto3_optional_ = false;
deterministic_output_ = false;
} }
bool CommandLineInterface::MakeProtoProtoPathRelative( bool CommandLineInterface::MakeProtoProtoPathRelative(
@ -1582,6 +1586,11 @@ CommandLineInterface::ParseArgumentStatus CommandLineInterface::ParseArguments(
<< std::endl; << std::endl;
return PARSE_ARGUMENT_FAIL; return PARSE_ARGUMENT_FAIL;
} }
if (mode_ != MODE_ENCODE && deterministic_output_) {
std::cerr << "Can only use --deterministic_output with --encode."
<< std::endl;
return PARSE_ARGUMENT_FAIL;
}
if (!dependency_out_name_.empty() && input_files_.size() > 1) { if (!dependency_out_name_.empty() && input_files_.size() > 1) {
std::cerr std::cerr
<< "Can only process one input file when using --dependency_out=FILE." << "Can only process one input file when using --dependency_out=FILE."
@ -1650,7 +1659,8 @@ bool CommandLineInterface::ParseArgument(const char* arg, std::string* name,
*name == "--include_imports" || *name == "--include_source_info" || *name == "--include_imports" || *name == "--include_source_info" ||
*name == "--version" || *name == "--decode_raw" || *name == "--version" || *name == "--decode_raw" ||
*name == "--print_free_field_numbers" || *name == "--print_free_field_numbers" ||
*name == "--experimental_allow_proto3_optional") { *name == "--experimental_allow_proto3_optional" ||
*name == "--deterministic_output") {
// HACK: These are the only flags that don't take a value. // HACK: These are the only flags that don't take a value.
// They probably should not be hard-coded like this but for now it's // They probably should not be hard-coded like this but for now it's
// not worth doing better. // not worth doing better.
@ -1857,6 +1867,7 @@ CommandLineInterface::InterpretArgument(const std::string& name,
} else if (name == "--disallow_services") { } else if (name == "--disallow_services") {
disallow_services_ = true; disallow_services_ = true;
} else if (name == "--experimental_allow_proto3_optional") { } else if (name == "--experimental_allow_proto3_optional") {
allow_proto3_optional_ = true; allow_proto3_optional_ = true;
@ -1890,6 +1901,9 @@ CommandLineInterface::InterpretArgument(const std::string& name,
codec_type_ = value; codec_type_ = value;
} else if (name == "--deterministic_output") {
deterministic_output_ = true;
} else if (name == "--error_format") { } else if (name == "--error_format") {
if (value == "gcc") { if (value == "gcc") {
error_format_ = ERROR_FORMAT_GCC; error_format_ = ERROR_FORMAT_GCC;
@ -2032,6 +2046,12 @@ void CommandLineInterface::PrintHelpText() {
"must\n" "must\n"
" be defined in PROTO_FILES or their " " be defined in PROTO_FILES or their "
"imports.\n" "imports.\n"
" --deterministic_output When using --encode, ensure map fields "
"are\n"
" deterministically ordered. Note that"
"this order is not\n"
" canonical, and changes across builds"
"or releases of protoc.\n"
" --decode=MESSAGE_TYPE Read a binary message of the given " " --decode=MESSAGE_TYPE Read a binary message of the given "
"type from\n" "type from\n"
" standard input and write it in text " " standard input and write it in text "
@ -2444,7 +2464,9 @@ bool CommandLineInterface::EncodeOrDecode(const DescriptorPool* pool) {
if (mode_ == MODE_ENCODE) { if (mode_ == MODE_ENCODE) {
// Output is binary. // Output is binary.
if (!message->SerializePartialToZeroCopyStream(&out)) { io::CodedOutputStream coded_out(&out);
coded_out.SetSerializationDeterministic(deterministic_output_);
if (!message->SerializePartialToCodedStream(&coded_out)) {
std::cerr << "output: I/O error." << std::endl; std::cerr << "output: I/O error." << std::endl;
return false; return false;
} }

@ -451,6 +451,9 @@ class PROTOC_EXPORT CommandLineInterface {
// Was the --experimental_allow_proto3_optional flag used? // Was the --experimental_allow_proto3_optional flag used?
bool allow_proto3_optional_ = false; bool allow_proto3_optional_ = false;
// When using --encode, this will be passed to SetSerializationDeterministic.
bool deterministic_output_ = false;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CommandLineInterface); GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CommandLineInterface);
}; };

@ -40,12 +40,14 @@
#include <unistd.h> #include <unistd.h>
#endif #endif
#include <memory> #include <memory>
#include <string>
#include <vector> #include <vector>
#include <google/protobuf/stubs/stringprintf.h> #include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/testing/file.h> #include <google/protobuf/testing/file.h>
#include <google/protobuf/testing/file.h> #include <google/protobuf/testing/file.h>
#include <google/protobuf/testing/file.h> #include <google/protobuf/testing/file.h>
#include <google/protobuf/any.pb.h>
#include <google/protobuf/compiler/mock_code_generator.h> #include <google/protobuf/compiler/mock_code_generator.h>
#include <google/protobuf/compiler/subprocess.h> #include <google/protobuf/compiler/subprocess.h>
#include <google/protobuf/compiler/code_generator.h> #include <google/protobuf/compiler/code_generator.h>
@ -59,10 +61,10 @@
#include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor.h>
#include <google/protobuf/testing/googletest.h> #include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/substitute.h> #include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/io/io_win32.h> #include <google/protobuf/io/io_win32.h>
#include <google/protobuf/stubs/strutil.h>
namespace google { namespace google {
namespace protobuf { namespace protobuf {
@ -680,6 +682,9 @@ TEST_F(CommandLineInterfaceTest, MultipleInputs_UnusedImport_DescriptorSetIn) {
FileDescriptorProto::descriptor()->file(); FileDescriptorProto::descriptor()->file();
descriptor_file->CopyTo(file_descriptor_set.add_file()); descriptor_file->CopyTo(file_descriptor_set.add_file());
FileDescriptorProto& any_proto = *file_descriptor_set.add_file();
google::protobuf::Any::descriptor()->file()->CopyTo(&any_proto);
const FileDescriptor* custom_file = const FileDescriptor* custom_file =
protobuf_unittest::AggregateMessage::descriptor()->file(); protobuf_unittest::AggregateMessage::descriptor()->file();
FileDescriptorProto* file_descriptor_proto = file_descriptor_set.add_file(); FileDescriptorProto* file_descriptor_proto = file_descriptor_set.add_file();
@ -1387,6 +1392,7 @@ TEST_F(CommandLineInterfaceTest, AllowServicesHasService) {
ExpectGenerated("test_generator", "", "foo.proto", "Foo"); ExpectGenerated("test_generator", "", "foo.proto", "Foo");
} }
TEST_F(CommandLineInterfaceTest, DirectDependencies_Missing_EmptyList) { TEST_F(CommandLineInterfaceTest, DirectDependencies_Missing_EmptyList) {
CreateTempFile("foo.proto", CreateTempFile("foo.proto",
"syntax = \"proto2\";\n" "syntax = \"proto2\";\n"
@ -2567,7 +2573,10 @@ class EncodeDecodeTest : public testing::TestWithParam<EncodeDecodeTestMode> {
bool Run(const std::string& command, bool specify_proto_files = true) { bool Run(const std::string& command, bool specify_proto_files = true) {
std::vector<std::string> args; std::vector<std::string> args;
args.push_back("protoc"); args.push_back("protoc");
SplitStringUsing(command, " ", &args); for (StringPiece split_piece :
Split(command, " ", true)) {
args.push_back(std::string(split_piece));
}
if (specify_proto_files) { if (specify_proto_files) {
switch (GetParam()) { switch (GetParam()) {
case PROTO_PATH: case PROTO_PATH:
@ -2726,6 +2735,32 @@ TEST_P(EncodeDecodeTest, ProtoParseError) {
"net/proto2/internal/no_such_file.proto: No such file or directory\n"); "net/proto2/internal/no_such_file.proto: No such file or directory\n");
} }
TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) {
RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/"
"testdata/text_format_unittest_data_oneof_implemented.txt"));
std::string args;
if (GetParam() != DESCRIPTOR_SET_IN) {
args.append(
TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto"));
}
EXPECT_TRUE(Run(
args + " --encode=protobuf_unittest.TestAllTypes --deterministic_output"));
ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented"));
ExpectStderrMatchesText("");
}
TEST_P(EncodeDecodeTest, DecodeDeterministicOutput) {
RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented"));
EXPECT_FALSE(
Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") +
" --decode=protobuf_unittest.TestAllTypes --deterministic_output"));
ExpectStderrMatchesText(
"Can only use --deterministic_output with --encode.\n");
}
INSTANTIATE_TEST_SUITE_P(FileDescriptorSetSource, EncodeDecodeTest, INSTANTIATE_TEST_SUITE_P(FileDescriptorSetSource, EncodeDecodeTest,
testing::Values(PROTO_PATH, DESCRIPTOR_SET_IN)); testing::Values(PROTO_PATH, DESCRIPTOR_SET_IN));
} // anonymous namespace } // anonymous namespace

@ -164,10 +164,6 @@ class FieldGenerator {
return false; return false;
} }
// Generate code that allocates the fields's default instance.
virtual void GenerateDefaultInstanceAllocator(
io::Printer* /*printer*/) const {}
// Generate lines to serialize this field directly to the array "target", // Generate lines to serialize this field directly to the array "target",
// which are placed within the message's SerializeWithCachedSizesToArray() // which are placed within the message's SerializeWithCachedSizesToArray()
// method. This must also advance "target" past the written bytes. // method. This must also advance "target" past the written bytes.
@ -178,11 +174,6 @@ class FieldGenerator {
// are placed in the message's ByteSize() method. // are placed in the message's ByteSize() method.
virtual void GenerateByteSize(io::Printer* printer) const = 0; virtual void GenerateByteSize(io::Printer* printer) const = 0;
// Any tags about field layout decisions (such as inlining) to embed in the
// offset.
virtual uint32 CalculateFieldTag() const { return 0; }
virtual bool IsInlined() const { return false; }
void SetHasBitIndex(int32 has_bit_index); void SetHasBitIndex(int32 has_bit_index);
protected: protected:

@ -941,7 +941,6 @@ void FileGenerator::GenerateInitForSCC(const SCC* scc,
if (scc_analyzer_.GetSCC(message_generators_[i]->descriptor_) != scc) { if (scc_analyzer_.GetSCC(message_generators_[i]->descriptor_) != scc) {
continue; continue;
} }
message_generators_[i]->GenerateFieldDefaultInstances(printer);
format( format(
"{\n" "{\n"
" void* ptr = &$1$;\n" " void* ptr = &$1$;\n"
@ -1198,7 +1197,6 @@ void FileGenerator::GenerateForwardDeclarations(io::Printer* printer) {
decls[Namespace(d, options_)].AddEnum(d); decls[Namespace(d, options_)].AddEnum(d);
} }
{ {
NamespaceOpener ns(format); NamespaceOpener ns(format);
for (const auto& pair : decls) { for (const auto& pair : decls) {
@ -1281,7 +1279,6 @@ void FileGenerator::GenerateLibraryIncludes(io::Printer* printer) {
IncludeFile("net/proto2/public/arenastring.h", printer); IncludeFile("net/proto2/public/arenastring.h", printer);
IncludeFile("net/proto2/public/generated_message_table_driven.h", printer); IncludeFile("net/proto2/public/generated_message_table_driven.h", printer);
IncludeFile("net/proto2/public/generated_message_util.h", printer); IncludeFile("net/proto2/public/generated_message_util.h", printer);
IncludeFile("net/proto2/public/inlined_string_field.h", printer);
IncludeFile("net/proto2/public/metadata_lite.h", printer); IncludeFile("net/proto2/public/metadata_lite.h", printer);
if (HasDescriptorMethods(file_, options_)) { if (HasDescriptorMethods(file_, options_)) {

@ -787,25 +787,6 @@ std::string SafeFunctionName(const Descriptor* descriptor,
return function_name; return function_name;
} }
bool IsStringInlined(const FieldDescriptor* descriptor,
const Options& options) {
if (options.opensource_runtime) return false;
// TODO(ckennelly): Handle inlining for any.proto.
if (IsAnyMessage(descriptor->containing_type(), options)) return false;
if (descriptor->containing_type()->options().map_entry()) return false;
// We rely on has bits to distinguish field presence for release_$name$. When
// there is no hasbit, we cannot use the address of the string instance when
// the field has been inlined.
if (!HasHasbit(descriptor)) return false;
if (options.access_info_map) {
if (descriptor->is_required()) return true;
}
return false;
}
static bool HasLazyFields(const Descriptor* descriptor, static bool HasLazyFields(const Descriptor* descriptor,
const Options& options) { const Options& options) {
for (int field_idx = 0; field_idx < descriptor->field_count(); field_idx++) { for (int field_idx = 0; field_idx < descriptor->field_count(); field_idx++) {
@ -1470,8 +1451,7 @@ class ParseLoopGenerator {
GetOptimizeFor(field->file(), options_) != FileOptions::LITE_RUNTIME && GetOptimizeFor(field->file(), options_) != FileOptions::LITE_RUNTIME &&
// For now only use arena string for strings with empty defaults. // For now only use arena string for strings with empty defaults.
field->default_value_string().empty() && field->default_value_string().empty() &&
!IsStringInlined(field, options_) && !field->real_containing_oneof() && !field->real_containing_oneof() && ctype == FieldOptions::STRING) {
ctype == FieldOptions::STRING) {
GenerateArenaString(field); GenerateArenaString(field);
} else { } else {
std::string name; std::string name;

@ -318,8 +318,6 @@ inline bool IsWeak(const FieldDescriptor* field, const Options& options) {
return false; return false;
} }
bool IsStringInlined(const FieldDescriptor* descriptor, const Options& options);
// For a string field, returns the effective ctype. If the actual ctype is // For a string field, returns the effective ctype. If the actual ctype is
// not supported, returns the default of STRING. // not supported, returns the default of STRING.
FieldOptions::CType EffectiveStringCType(const FieldDescriptor* field, FieldOptions::CType EffectiveStringCType(const FieldDescriptor* field,
@ -410,14 +408,6 @@ inline bool IsProto2MessageSet(const Descriptor* descriptor,
descriptor->full_name() == "google.protobuf.bridge.MessageSet"; descriptor->full_name() == "google.protobuf.bridge.MessageSet";
} }
inline bool IsProto2MessageSetFile(const FileDescriptor* file,
const Options& options) {
return !options.opensource_runtime &&
options.enforce_mode != EnforceOptimizeMode::kLiteRuntime &&
!options.lite_implicit_weak_fields &&
file->name() == "net/proto2/bridge/proto/message_set.proto";
}
inline bool IsMapEntryMessage(const Descriptor* descriptor) { inline bool IsMapEntryMessage(const Descriptor* descriptor) {
return descriptor->options().map_entry(); return descriptor->options().map_entry();
} }

@ -1699,9 +1699,6 @@ uint32 CalcFieldNum(const FieldGenerator& generator,
int type = field->type(); int type = field->type();
if (type == FieldDescriptor::TYPE_STRING || if (type == FieldDescriptor::TYPE_STRING ||
type == FieldDescriptor::TYPE_BYTES) { type == FieldDescriptor::TYPE_BYTES) {
if (generator.IsInlined()) {
type = internal::FieldMetadata::kInlinedType;
}
// string field // string field
if (IsCord(field, options)) { if (IsCord(field, options)) {
type = internal::FieldMetadata::kCordType; type = internal::FieldMetadata::kCordType;
@ -1902,13 +1899,6 @@ int MessageGenerator::GenerateFieldMetadata(io::Printer* printer) {
return num_field_metadata; return num_field_metadata;
} }
void MessageGenerator::GenerateFieldDefaultInstances(io::Printer* printer) {
// Construct the default instances for all fields that need one.
for (auto field : FieldRange(descriptor_)) {
field_generators_.get(field).GenerateDefaultInstanceAllocator(printer);
}
}
void MessageGenerator::GenerateClassMethods(io::Printer* printer) { void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
Formatter format(printer, variables_); Formatter format(printer, variables_);
if (IsMapEntryMessage(descriptor_)) { if (IsMapEntryMessage(descriptor_)) {
@ -2111,14 +2101,9 @@ size_t MessageGenerator::GenerateParseOffsets(io::Printer* printer) {
} }
processing_type = static_cast<unsigned>(field->type()); processing_type = static_cast<unsigned>(field->type());
const FieldGenerator& generator = field_generators_.get(field);
if (field->type() == FieldDescriptor::TYPE_STRING) { if (field->type() == FieldDescriptor::TYPE_STRING) {
switch (EffectiveStringCType(field, options_)) { switch (EffectiveStringCType(field, options_)) {
case FieldOptions::STRING: case FieldOptions::STRING:
if (generator.IsInlined()) {
processing_type = internal::TYPE_STRING_INLINED;
break;
}
break; break;
case FieldOptions::CORD: case FieldOptions::CORD:
processing_type = internal::TYPE_STRING_CORD; processing_type = internal::TYPE_STRING_CORD;
@ -2130,10 +2115,6 @@ size_t MessageGenerator::GenerateParseOffsets(io::Printer* printer) {
} else if (field->type() == FieldDescriptor::TYPE_BYTES) { } else if (field->type() == FieldDescriptor::TYPE_BYTES) {
switch (EffectiveStringCType(field, options_)) { switch (EffectiveStringCType(field, options_)) {
case FieldOptions::STRING: case FieldOptions::STRING:
if (generator.IsInlined()) {
processing_type = internal::TYPE_BYTES_INLINED;
break;
}
break; break;
case FieldOptions::CORD: case FieldOptions::CORD:
processing_type = internal::TYPE_BYTES_CORD; processing_type = internal::TYPE_BYTES_CORD;
@ -2317,11 +2298,6 @@ std::pair<size_t, size_t> MessageGenerator::GenerateOffsets(
format("PROTOBUF_FIELD_OFFSET($classtype$, $1$_)", FieldName(field)); format("PROTOBUF_FIELD_OFFSET($classtype$, $1$_)", FieldName(field));
} }
uint32 tag = field_generators_.get(field).CalculateFieldTag();
if (tag != 0) {
format(" | $1$", tag);
}
if (!IsFieldUsed(field, options_)) { if (!IsFieldUsed(field, options_)) {
format(" | 0x80000000u, // unused\n"); format(" | 0x80000000u, // unused\n");
} else { } else {

@ -82,9 +82,6 @@ class MessageGenerator {
// Source file stuff. // Source file stuff.
// Generates code that creates default instances for fields.
void GenerateFieldDefaultInstances(io::Printer* printer);
// Generate all non-inline methods for this class. // Generate all non-inline methods for this class.
void GenerateClassMethods(io::Printer* printer); void GenerateClassMethods(io::Printer* printer);

@ -55,12 +55,31 @@ void SetStringVariables(const FieldDescriptor* descriptor,
StrCat(descriptor->default_value_string().length()); StrCat(descriptor->default_value_string().length());
std::string default_variable_string = MakeDefaultName(descriptor); std::string default_variable_string = MakeDefaultName(descriptor);
(*variables)["default_variable_name"] = default_variable_string; (*variables)["default_variable_name"] = default_variable_string;
(*variables)["default_variable"] =
if (!descriptor->default_value_string().empty()) {
(*variables)["lazy_variable"] =
QualifiedClassName(descriptor->containing_type(), options) +
"::" + default_variable_string;
}
(*variables)["default_string"] =
descriptor->default_value_string().empty()
? "::" + (*variables)["proto_ns"] +
"::internal::GetEmptyStringAlreadyInited()"
: (*variables)["lazy_variable"] + ".get()";
(*variables)["init_value"] =
descriptor->default_value_string().empty() descriptor->default_value_string().empty()
? "&::" + (*variables)["proto_ns"] + ? "&::" + (*variables)["proto_ns"] +
"::internal::GetEmptyStringAlreadyInited()" "::internal::GetEmptyStringAlreadyInited()"
: "&" + QualifiedClassName(descriptor->containing_type(), options) + : "nullptr";
"::" + default_variable_string + ".get()"; (*variables)["default_value_tag"] =
"::" + (*variables)["proto_ns"] + "::internal::ArenaStringPtr::" +
(descriptor->default_value_string().empty() ? "Empty" : "NonEmpty") +
"Default{}";
(*variables)["default_variable_or_tag"] =
(*variables)[descriptor->default_value_string().empty()
? "default_value_tag"
: "lazy_variable"];
(*variables)["pointer_type"] = (*variables)["pointer_type"] =
descriptor->type() == FieldDescriptor::TYPE_BYTES ? "void" : "char"; descriptor->type() == FieldDescriptor::TYPE_BYTES ? "void" : "char";
(*variables)["null_check"] = (*variables)["DCHK"] + "(value != nullptr);\n"; (*variables)["null_check"] = (*variables)["DCHK"] + "(value != nullptr);\n";
@ -75,9 +94,6 @@ void SetStringVariables(const FieldDescriptor* descriptor,
} else { } else {
(*variables)["string_piece"] = "::StringPiece"; (*variables)["string_piece"] = "::StringPiece";
} }
(*variables)["lite"] =
HasDescriptorMethods(descriptor->file(), options) ? "" : "Lite";
} }
} // namespace } // namespace
@ -86,9 +102,7 @@ void SetStringVariables(const FieldDescriptor* descriptor,
StringFieldGenerator::StringFieldGenerator(const FieldDescriptor* descriptor, StringFieldGenerator::StringFieldGenerator(const FieldDescriptor* descriptor,
const Options& options) const Options& options)
: FieldGenerator(descriptor, options), : FieldGenerator(descriptor, options) {
lite_(!HasDescriptorMethods(descriptor->file(), options)),
inlined_(IsStringInlined(descriptor, options)) {
SetStringVariables(descriptor, &variables_, options); SetStringVariables(descriptor, &variables_, options);
} }
@ -96,23 +110,15 @@ StringFieldGenerator::~StringFieldGenerator() {}
void StringFieldGenerator::GeneratePrivateMembers(io::Printer* printer) const { void StringFieldGenerator::GeneratePrivateMembers(io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
if (inlined_) {
format("::$proto_ns$::internal::InlinedStringField $name$_;\n");
} else {
format("::$proto_ns$::internal::ArenaStringPtr $name$_;\n"); format("::$proto_ns$::internal::ArenaStringPtr $name$_;\n");
}
} }
void StringFieldGenerator::GenerateStaticMembers(io::Printer* printer) const { void StringFieldGenerator::GenerateStaticMembers(io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) { if (!descriptor_->default_value_string().empty()) {
// We make the default instance public, so it can be initialized by
// non-friend code.
format( format(
"public:\n" "static const ::$proto_ns$::internal::LazyString"
"static ::$proto_ns$::internal::ExplicitlyConstructed<std::string>" " $default_variable_name$;\n");
" $default_variable_name$;\n"
"private:\n");
} }
} }
@ -187,7 +193,13 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
format( format(
"inline const std::string& $classname$::$name$() const {\n" "inline const std::string& $classname$::$name$() const {\n"
"$annotate_accessor$" "$annotate_accessor$"
" // @@protoc_insertion_point(field_get:$full_name$)\n" " // @@protoc_insertion_point(field_get:$full_name$)\n");
if (!descriptor_->default_value_string().empty()) {
format(
" if ($name$_.IsDefault(nullptr)) return "
"$default_variable_name$.get();\n");
}
format(
" return _internal_$name$();\n" " return _internal_$name$();\n"
"}\n" "}\n"
"inline void $classname$::set_$name$(const std::string& value) {\n" "inline void $classname$::set_$name$(const std::string& value) {\n"
@ -206,21 +218,20 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
"inline void $classname$::_internal_set_$name$(const std::string& " "inline void $classname$::_internal_set_$name$(const std::string& "
"value) {\n" "value) {\n"
" $set_hasbit$\n" " $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, value, GetArena());\n" " $name$_.Set($default_value_tag$, value, GetArena());\n"
"}\n" "}\n"
"inline void $classname$::set_$name$(std::string&& value) {\n" "inline void $classname$::set_$name$(std::string&& value) {\n"
"$annotate_accessor$" "$annotate_accessor$"
" $set_hasbit$\n" " $set_hasbit$\n"
" $name$_.Set$lite$(\n" " $name$_.Set(\n"
" $default_variable$, ::std::move(value), GetArena());\n" " $default_value_tag$, ::std::move(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n" " // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
"}\n" "}\n"
"inline void $classname$::set_$name$(const char* value) {\n" "inline void $classname$::set_$name$(const char* value) {\n"
"$annotate_accessor$" "$annotate_accessor$"
" $null_check$" " $null_check$"
" $set_hasbit$\n" " $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, $string_piece$(value),\n" " $name$_.Set($default_value_tag$, $string_piece$(value), GetArena());\n"
" GetArena());\n"
" // @@protoc_insertion_point(field_set_char:$full_name$)\n" " // @@protoc_insertion_point(field_set_char:$full_name$)\n"
"}\n"); "}\n");
if (!options_.opensource_runtime) { if (!options_.opensource_runtime) {
@ -228,7 +239,7 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
"inline void $classname$::set_$name$(::StringPiece value) {\n" "inline void $classname$::set_$name$(::StringPiece value) {\n"
"$annotate_accessor$" "$annotate_accessor$"
" $set_hasbit$\n" " $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, value,GetArena());\n" " $name$_.Set($default_value_tag$, value,GetArena());\n"
" // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n" " // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n"
"}\n"); "}\n");
} }
@ -238,13 +249,13 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
" size_t size) {\n" " size_t size) {\n"
"$annotate_accessor$" "$annotate_accessor$"
" $set_hasbit$\n" " $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, $string_piece$(\n" " $name$_.Set($default_value_tag$, $string_piece$(\n"
" reinterpret_cast<const char*>(value), size), GetArena());\n" " reinterpret_cast<const char*>(value), size), GetArena());\n"
" // @@protoc_insertion_point(field_set_pointer:$full_name$)\n" " // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
"}\n" "}\n"
"inline std::string* $classname$::_internal_mutable_$name$() {\n" "inline std::string* $classname$::_internal_mutable_$name$() {\n"
" $set_hasbit$\n" " $set_hasbit$\n"
" return $name$_.Mutable($default_variable$, GetArena());\n" " return $name$_.Mutable($default_variable_or_tag$, GetArena());\n"
"}\n" "}\n"
"inline std::string* $classname$::$release_name$() {\n" "inline std::string* $classname$::$release_name$() {\n"
"$annotate_accessor$" "$annotate_accessor$"
@ -256,10 +267,9 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
" return nullptr;\n" " return nullptr;\n"
" }\n" " }\n"
" $clear_hasbit$\n" " $clear_hasbit$\n"
" return $name$_.ReleaseNonDefault(" " return $name$_.ReleaseNonDefault($init_value$, GetArena());\n");
"$default_variable$, GetArena());\n");
} else { } else {
format(" return $name$_.Release($default_variable$, GetArena());\n"); format(" return $name$_.Release($init_value$, GetArena());\n");
} }
format( format(
@ -271,7 +281,7 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
" } else {\n" " } else {\n"
" $clear_hasbit$\n" " $clear_hasbit$\n"
" }\n" " }\n"
" $name$_.SetAllocated($default_variable$, $name$,\n" " $name$_.SetAllocated($init_value$, $name$,\n"
" GetArena());\n" " GetArena());\n"
" // @@protoc_insertion_point(field_set_allocated:$full_name$)\n" " // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
"}\n"); "}\n");
@ -281,23 +291,19 @@ void StringFieldGenerator::GenerateNonInlineAccessorDefinitions(
io::Printer* printer) const { io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) { if (!descriptor_->default_value_string().empty()) {
// Initialized in GenerateDefaultInstanceAllocator.
format( format(
"::$proto_ns$::internal::ExplicitlyConstructed<std::string> " "const ::$proto_ns$::internal::LazyString "
"$classname$::$default_variable_name$;\n"); "$classname$::$default_variable_name$"
"{{{$default$, $default_length$}}, {nullptr}};\n");
} }
} }
void StringFieldGenerator::GenerateClearingCode(io::Printer* printer) const { void StringFieldGenerator::GenerateClearingCode(io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
// Two-dimension specialization here: supporting arenas or not, and default
// value is the empty string or not. Complexity here ensures the minimal
// number of branches / amount of extraneous code at runtime (given that the
// below methods are inlined one-liners)!
if (descriptor_->default_value_string().empty()) { if (descriptor_->default_value_string().empty()) {
format("$name$_.ClearToEmpty($default_variable$, GetArena());\n"); format("$name$_.ClearToEmpty();\n");
} else { } else {
format("$name$_.ClearToDefault($default_variable$, GetArena());\n"); format("$name$_.ClearToDefault($lazy_variable$, GetArena());\n");
} }
} }
@ -311,31 +317,19 @@ void StringFieldGenerator::GenerateMessageClearingCode(
// If we have a hasbit, then the Clear() method of the protocol buffer // If we have a hasbit, then the Clear() method of the protocol buffer
// will have checked that this field is set. If so, we can avoid redundant // will have checked that this field is set. If so, we can avoid redundant
// checks against default_variable. // checks against the default variable.
const bool must_be_present = HasHasbit(descriptor_); const bool must_be_present = HasHasbit(descriptor_);
if (inlined_ && must_be_present) {
// Calling mutable_$name$() gives us a string reference and sets the has bit
// for $name$ (in proto2). We may get here when the string field is inlined
// but the string's contents have not been changed by the user, so we cannot
// make an assertion about the contents of the string and could never make
// an assertion about the string instance.
//
// For non-inlined strings, we distinguish from non-default by comparing
// instances, rather than contents.
format("$DCHK$(!$name$_.IsDefault($default_variable$));\n");
}
if (descriptor_->default_value_string().empty()) { if (descriptor_->default_value_string().empty()) {
if (must_be_present) { if (must_be_present) {
format("$name$_.ClearNonDefaultToEmpty();\n"); format("$name$_.ClearNonDefaultToEmpty();\n");
} else { } else {
format("$name$_.ClearToEmpty($default_variable$, GetArena());\n"); format("$name$_.ClearToEmpty();\n");
} }
} else { } else {
// Clear to a non-empty default is more involved, as we try to use the // Clear to a non-empty default is more involved, as we try to use the
// Arena if one is present and may need to reallocate the string. // Arena if one is present and may need to reallocate the string.
format("$name$_.ClearToDefault($default_variable$, GetArena());\n"); format("$name$_.ClearToDefault($lazy_variable$, GetArena());\n ");
} }
} }
@ -347,23 +341,12 @@ void StringFieldGenerator::GenerateMergingCode(io::Printer* printer) const {
void StringFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { void StringFieldGenerator::GenerateSwappingCode(io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
if (inlined_) { format("$name$_.Swap(&other->$name$_, $init_value$, GetArena());\n");
format("$name$_.Swap(&other->$name$_);\n");
} else {
format("$name$_.Swap(&other->$name$_, $default_variable$, GetArena());\n");
}
} }
void StringFieldGenerator::GenerateConstructorCode(io::Printer* printer) const { void StringFieldGenerator::GenerateConstructorCode(io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
// TODO(ckennelly): Construct non-empty strings as part of the initializer format("$name$_.UnsafeSetDefault($init_value$);\n");
// list.
if (inlined_ && descriptor_->default_value_string().empty()) {
// Automatic initialization will construct the string.
return;
}
format("$name$_.UnsafeSetDefault($default_variable$);\n");
} }
void StringFieldGenerator::GenerateCopyConstructorCode( void StringFieldGenerator::GenerateCopyConstructorCode(
@ -381,7 +364,7 @@ void StringFieldGenerator::GenerateCopyConstructorCode(
// TODO(gpike): improve this // TODO(gpike): improve this
format( format(
"$name$_.Set$lite$($default_variable$, from._internal_$name$(),\n" "$name$_.Set($default_value_tag$, from._internal_$name$(), \n"
" GetArena());\n"); " GetArena());\n");
format.Outdent(); format.Outdent();
@ -390,40 +373,7 @@ void StringFieldGenerator::GenerateCopyConstructorCode(
void StringFieldGenerator::GenerateDestructorCode(io::Printer* printer) const { void StringFieldGenerator::GenerateDestructorCode(io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
if (inlined_) { format("$name$_.DestroyNoArena($init_value$);\n");
// The destructor is automatically invoked.
return;
}
format("$name$_.DestroyNoArena($default_variable$);\n");
}
bool StringFieldGenerator::GenerateArenaDestructorCode(
io::Printer* printer) const {
Formatter format(printer, variables_);
if (!inlined_) {
return false;
}
format("_this->$name$_.DestroyNoArena($default_variable$);\n");
return true;
}
void StringFieldGenerator::GenerateDefaultInstanceAllocator(
io::Printer* printer) const {
Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) {
format(
"$ns$::$classname$::$default_variable_name$.DefaultConstruct();\n"
"*$ns$::$classname$::$default_variable_name$.get_mutable() = "
"std::string($default$, $default_length$);\n"
"::$proto_ns$::internal::OnShutdownDestroyString(\n"
" $ns$::$classname$::$default_variable_name$.get_mutable());\n");
}
}
bool StringFieldGenerator::MergeFromCodedStreamNeedsArena() const {
return !lite_ && !inlined_ && !options_.opensource_runtime;
} }
void StringFieldGenerator::GenerateSerializeWithCachedSizesToArray( void StringFieldGenerator::GenerateSerializeWithCachedSizesToArray(
@ -449,17 +399,11 @@ void StringFieldGenerator::GenerateByteSize(io::Printer* printer) const {
" this->_internal_$name$());\n"); " this->_internal_$name$());\n");
} }
uint32 StringFieldGenerator::CalculateFieldTag() const {
return inlined_ ? 1 : 0;
}
// =================================================================== // ===================================================================
StringOneofFieldGenerator::StringOneofFieldGenerator( StringOneofFieldGenerator::StringOneofFieldGenerator(
const FieldDescriptor* descriptor, const Options& options) const FieldDescriptor* descriptor, const Options& options)
: StringFieldGenerator(descriptor, options) { : StringFieldGenerator(descriptor, options) {
inlined_ = false;
SetCommonOneofFieldVariables(descriptor, &variables_); SetCommonOneofFieldVariables(descriptor, &variables_);
variables_["field_name"] = UnderscoresToCamelCase(descriptor->name(), true); variables_["field_name"] = UnderscoresToCamelCase(descriptor->name(), true);
variables_["oneof_index"] = variables_["oneof_index"] =
@ -491,16 +435,16 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (_internal_has_$name$()) {\n" " if (_internal_has_$name$()) {\n"
" return $field_member$.Get();\n" " return $field_member$.Get();\n"
" }\n" " }\n"
" return *$default_variable$;\n" " return $default_string$;\n"
"}\n" "}\n"
"inline void $classname$::_internal_set_$name$(const std::string& " "inline void $classname$::_internal_set_$name$(const std::string& "
"value) {\n" "value) {\n"
" if (!_internal_has_$name$()) {\n" " if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n" " clear_$oneof_name$();\n"
" set_has_$name$();\n" " set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n" " $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n" " }\n"
" $field_member$.Set$lite$($default_variable$, value, GetArena());\n" " $field_member$.Set($default_value_tag$, value, GetArena());\n"
"}\n" "}\n"
"inline void $classname$::set_$name$(std::string&& value) {\n" "inline void $classname$::set_$name$(std::string&& value) {\n"
"$annotate_accessor$" "$annotate_accessor$"
@ -508,10 +452,10 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n" " if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n" " clear_$oneof_name$();\n"
" set_has_$name$();\n" " set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n" " $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n" " }\n"
" $field_member$.Set$lite$(\n" " $field_member$.Set(\n"
" $default_variable$, ::std::move(value), GetArena());\n" " $default_value_tag$, ::std::move(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n" " // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
"}\n" "}\n"
"inline void $classname$::set_$name$(const char* value) {\n" "inline void $classname$::set_$name$(const char* value) {\n"
@ -520,9 +464,9 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n" " if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n" " clear_$oneof_name$();\n"
" set_has_$name$();\n" " set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n" " $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n" " }\n"
" $field_member$.Set$lite$($default_variable$,\n" " $field_member$.Set($default_value_tag$,\n"
" $string_piece$(value), GetArena());\n" " $string_piece$(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_char:$full_name$)\n" " // @@protoc_insertion_point(field_set_char:$full_name$)\n"
"}\n"); "}\n");
@ -533,10 +477,9 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n" " if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n" " clear_$oneof_name$();\n"
" set_has_$name$();\n" " set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n" " $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n" " }\n"
" $field_member$.Set$lite$($default_variable$, value,\n" " $field_member$.Set($default_value_tag$, value, GetArena());\n"
" GetArena());\n"
" // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n" " // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n"
"}\n"); "}\n");
} }
@ -548,10 +491,10 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n" " if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n" " clear_$oneof_name$();\n"
" set_has_$name$();\n" " set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n" " $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n" " }\n"
" $field_member$.Set$lite$(\n" " $field_member$.Set(\n"
" $default_variable$, $string_piece$(\n" " $default_value_tag$, $string_piece$(\n"
" reinterpret_cast<const char*>(value), size),\n" " reinterpret_cast<const char*>(value), size),\n"
" GetArena());\n" " GetArena());\n"
" // @@protoc_insertion_point(field_set_pointer:$full_name$)\n" " // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
@ -560,16 +503,17 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n" " if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n" " clear_$oneof_name$();\n"
" set_has_$name$();\n" " set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n" " $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n" " }\n"
" return $field_member$.Mutable($default_variable$, GetArena());\n" " return $field_member$.Mutable(\n"
" $default_variable_or_tag$, GetArena());\n"
"}\n" "}\n"
"inline std::string* $classname$::$release_name$() {\n" "inline std::string* $classname$::$release_name$() {\n"
"$annotate_accessor$" "$annotate_accessor$"
" // @@protoc_insertion_point(field_release:$full_name$)\n" " // @@protoc_insertion_point(field_release:$full_name$)\n"
" if (_internal_has_$name$()) {\n" " if (_internal_has_$name$()) {\n"
" clear_has_$oneof_name$();\n" " clear_has_$oneof_name$();\n"
" return $field_member$.Release($default_variable$, GetArena());\n" " return $field_member$.ReleaseNonDefault($init_value$, GetArena());\n"
" } else {\n" " } else {\n"
" return nullptr;\n" " return nullptr;\n"
" }\n" " }\n"
@ -594,7 +538,7 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
void StringOneofFieldGenerator::GenerateClearingCode( void StringOneofFieldGenerator::GenerateClearingCode(
io::Printer* printer) const { io::Printer* printer) const {
Formatter format(printer, variables_); Formatter format(printer, variables_);
format("$field_member$.Destroy($default_variable$, GetArena());\n"); format("$field_member$.Destroy($default_value_tag$, GetArena());\n");
} }
void StringOneofFieldGenerator::GenerateMessageClearingCode( void StringOneofFieldGenerator::GenerateMessageClearingCode(
@ -609,10 +553,7 @@ void StringOneofFieldGenerator::GenerateSwappingCode(
void StringOneofFieldGenerator::GenerateConstructorCode( void StringOneofFieldGenerator::GenerateConstructorCode(
io::Printer* printer) const { io::Printer* printer) const {
Formatter format(printer, variables_); // Nothing required here.
format(
"$ns$::_$classname$_default_instance_.$name$_.UnsafeSetDefault(\n"
" $default_variable$);\n");
} }
// =================================================================== // ===================================================================

@ -63,18 +63,8 @@ class StringFieldGenerator : public FieldGenerator {
void GenerateConstructorCode(io::Printer* printer) const; void GenerateConstructorCode(io::Printer* printer) const;
void GenerateCopyConstructorCode(io::Printer* printer) const; void GenerateCopyConstructorCode(io::Printer* printer) const;
void GenerateDestructorCode(io::Printer* printer) const; void GenerateDestructorCode(io::Printer* printer) const;
bool GenerateArenaDestructorCode(io::Printer* printer) const;
void GenerateDefaultInstanceAllocator(io::Printer* printer) const;
void GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const; void GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const;
void GenerateByteSize(io::Printer* printer) const; void GenerateByteSize(io::Printer* printer) const;
uint32 CalculateFieldTag() const;
bool IsInlined() const { return inlined_; }
bool MergeFromCodedStreamNeedsArena() const;
protected:
const bool lite_;
bool inlined_;
private: private:
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringFieldGenerator); GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringFieldGenerator);

@ -77,9 +77,10 @@ void EnumGenerator::Generate(io::Printer* printer) {
WriteEnumDocComment(printer, descriptor_); WriteEnumDocComment(printer, descriptor_);
MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_); MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_);
printer->Print( printer->Print(
"public enum $classname$\n" "$deprecation$public enum $classname$\n"
" implements com.google.protobuf.ProtocolMessageEnum {\n", " implements com.google.protobuf.ProtocolMessageEnum {\n",
"classname", descriptor_->name()); "classname", descriptor_->name(), "deprecation",
descriptor_->options().deprecated() ? "@java.lang.Deprecated " : "");
printer->Annotate("classname", descriptor_); printer->Annotate("classname", descriptor_);
printer->Indent(); printer->Indent();

@ -81,7 +81,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
// with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations. // with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations.
(*variables)["for_number"] = "valueOf"; (*variables)["for_number"] = "valueOf";
if (SupportFieldPresence(descriptor)) { if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit. // For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
(*variables)["get_has_field_bit_builder"] = GenerateGetBit(builderBitIndex); (*variables)["get_has_field_bit_builder"] = GenerateGetBit(builderBitIndex);
@ -145,7 +145,7 @@ ImmutableEnumFieldGenerator::ImmutableEnumFieldGenerator(
ImmutableEnumFieldGenerator::~ImmutableEnumFieldGenerator() {} ImmutableEnumFieldGenerator::~ImmutableEnumFieldGenerator() {}
int ImmutableEnumFieldGenerator::GetNumBitsForMessage() const { int ImmutableEnumFieldGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0; return HasHasbit(descriptor_) ? 1 : 0;
} }
int ImmutableEnumFieldGenerator::GetNumBitsForBuilder() const { int ImmutableEnumFieldGenerator::GetNumBitsForBuilder() const {

@ -83,7 +83,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
descriptor->options().deprecated() ? "@java.lang.Deprecated " : ""; descriptor->options().deprecated() ? "@java.lang.Deprecated " : "";
(*variables)["required"] = descriptor->is_required() ? "true" : "false"; (*variables)["required"] = descriptor->is_required() ? "true" : "false";
if (SupportFieldPresence(descriptor)) { if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit. // For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
@ -137,7 +137,7 @@ ImmutableEnumFieldLiteGenerator::ImmutableEnumFieldLiteGenerator(
ImmutableEnumFieldLiteGenerator::~ImmutableEnumFieldLiteGenerator() {} ImmutableEnumFieldLiteGenerator::~ImmutableEnumFieldLiteGenerator() {}
int ImmutableEnumFieldLiteGenerator::GetNumBitsForMessage() const { int ImmutableEnumFieldLiteGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0; return HasHasbit(descriptor_) ? 1 : 0;
} }
void ImmutableEnumFieldLiteGenerator::GenerateInterfaceMembers( void ImmutableEnumFieldLiteGenerator::GenerateInterfaceMembers(

@ -78,9 +78,10 @@ void EnumLiteGenerator::Generate(io::Printer* printer) {
WriteEnumDocComment(printer, descriptor_); WriteEnumDocComment(printer, descriptor_);
MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_); MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_);
printer->Print( printer->Print(
"public enum $classname$\n" "$deprecation$public enum $classname$\n"
" implements com.google.protobuf.Internal.EnumLite {\n", " implements com.google.protobuf.Internal.EnumLite {\n",
"classname", descriptor_->name()); "classname", descriptor_->name(), "deprecation",
descriptor_->options().deprecated() ? "@java.lang.Deprecated " : "");
printer->Annotate("classname", descriptor_); printer->Annotate("classname", descriptor_);
printer->Indent(); printer->Indent();

@ -76,7 +76,6 @@ struct FieldDescriptorCompare {
typedef std::set<const FieldDescriptor*, FieldDescriptorCompare> typedef std::set<const FieldDescriptor*, FieldDescriptorCompare>
FieldDescriptorSet; FieldDescriptorSet;
// Recursively searches the given message to collect extensions. // Recursively searches the given message to collect extensions.
// Returns true if all the extensions can be recognized. The extensions will be // Returns true if all the extensions can be recognized. The extensions will be
// appended in to the extensions parameter. // appended in to the extensions parameter.
@ -86,9 +85,7 @@ bool CollectExtensions(const Message& message, FieldDescriptorSet* extensions) {
const Reflection* reflection = message.GetReflection(); const Reflection* reflection = message.GetReflection();
// There are unknown fields that could be extensions, thus this call fails. // There are unknown fields that could be extensions, thus this call fails.
UnknownFieldSet unknown_fields; if (reflection->GetUnknownFields(message).field_count() > 0) return false;
unknown_fields.MergeFrom(reflection->GetUnknownFields(message));
if (unknown_fields.field_count() > 0) return false;
std::vector<const FieldDescriptor*> fields; std::vector<const FieldDescriptor*> fields;
reflection->ListFields(message, &fields); reflection->ListFields(message, &fields);
@ -390,6 +387,7 @@ void FileGenerator::Generate(io::Printer* printer) {
printer->Print("}\n"); printer->Print("}\n");
} }
void FileGenerator::GenerateDescriptorInitializationCodeForImmutable( void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
io::Printer* printer) { io::Printer* printer) {
printer->Print( printer->Print(

@ -78,6 +78,7 @@ class FileGenerator {
void Generate(io::Printer* printer); void Generate(io::Printer* printer);
// If we aren't putting everything into one file, this will write all the // If we aren't putting everything into one file, this will write all the
// files other than the outer file (i.e. one for each message, enum, and // files other than the outer file (i.e. one for each message, enum, and
// service type). // service type).

@ -1355,7 +1355,6 @@ void ImmutableMessageGenerator::GenerateInitializers(io::Printer* printer) {
} }
} }
void ImmutableMessageGenerator::GenerateAnyMethods(io::Printer* printer) { void ImmutableMessageGenerator::GenerateAnyMethods(io::Printer* printer) {
printer->Print( printer->Print(
"private static String getTypeUrl(\n" "private static String getTypeUrl(\n"

@ -160,7 +160,14 @@ std::string ClassNameResolver::GetFileImmutableClassName(
std::string ClassNameResolver::GetFileClassName(const FileDescriptor* file, std::string ClassNameResolver::GetFileClassName(const FileDescriptor* file,
bool immutable) { bool immutable) {
if (immutable) { return GetFileClassName(file, immutable, false);
}
std::string ClassNameResolver::GetFileClassName(const FileDescriptor* file,
bool immutable, bool kotlin) {
if (kotlin) {
return GetFileImmutableClassName(file) + "Kt";
} else if (immutable) {
return GetFileImmutableClassName(file); return GetFileImmutableClassName(file);
} else { } else {
return "Mutable" + GetFileImmutableClassName(file); return "Mutable" + GetFileImmutableClassName(file);
@ -200,9 +207,14 @@ std::string ClassNameResolver::GetDescriptorClassName(
std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor, std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
bool immutable) { bool immutable) {
return GetClassName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
bool immutable, bool kotlin) {
std::string result = FileJavaPackage(descriptor, immutable); std::string result = FileJavaPackage(descriptor, immutable);
if (!result.empty()) result += '.'; if (!result.empty()) result += '.';
result += GetFileClassName(descriptor, immutable); result += GetFileClassName(descriptor, immutable, kotlin);
return result; return result;
} }
@ -211,50 +223,79 @@ std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
std::string ClassNameResolver::GetClassFullName( std::string ClassNameResolver::GetClassFullName(
const std::string& name_without_package, const FileDescriptor* file, const std::string& name_without_package, const FileDescriptor* file,
bool immutable, bool is_own_file) { bool immutable, bool is_own_file) {
return GetClassFullName(name_without_package, file, immutable, is_own_file,
false);
}
std::string ClassNameResolver::GetClassFullName(
const std::string& name_without_package, const FileDescriptor* file,
bool immutable, bool is_own_file, bool kotlin) {
std::string result; std::string result;
if (is_own_file) { if (is_own_file) {
result = FileJavaPackage(file, immutable); result = FileJavaPackage(file, immutable);
} else { } else {
result = GetClassName(file, immutable); result = GetClassName(file, immutable, kotlin);
} }
if (!result.empty()) { if (!result.empty()) {
result += '.'; result += '.';
} }
result += name_without_package; result += name_without_package;
if (kotlin) result += "Kt";
return result; return result;
} }
std::string ClassNameResolver::GetClassName(const Descriptor* descriptor, std::string ClassNameResolver::GetClassName(const Descriptor* descriptor,
bool immutable) { bool immutable) {
return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable), return GetClassName(descriptor, immutable, false);
descriptor->file(), immutable, }
MultipleJavaFiles(descriptor->file(), immutable));
std::string ClassNameResolver::GetClassName(const Descriptor* descriptor,
bool immutable, bool kotlin) {
return GetClassFullName(
ClassNameWithoutPackage(descriptor, immutable), descriptor->file(),
immutable, MultipleJavaFiles(descriptor->file(), immutable), kotlin);
} }
std::string ClassNameResolver::GetClassName(const EnumDescriptor* descriptor, std::string ClassNameResolver::GetClassName(const EnumDescriptor* descriptor,
bool immutable) { bool immutable) {
return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable), return GetClassName(descriptor, immutable, false);
descriptor->file(), immutable, }
MultipleJavaFiles(descriptor->file(), immutable));
std::string ClassNameResolver::GetClassName(const EnumDescriptor* descriptor,
bool immutable, bool kotlin) {
return GetClassFullName(
ClassNameWithoutPackage(descriptor, immutable), descriptor->file(),
immutable, MultipleJavaFiles(descriptor->file(), immutable), kotlin);
} }
std::string ClassNameResolver::GetClassName(const ServiceDescriptor* descriptor, std::string ClassNameResolver::GetClassName(const ServiceDescriptor* descriptor,
bool immutable) { bool immutable) {
return GetClassName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetClassName(const ServiceDescriptor* descriptor,
bool immutable, bool kotlin) {
return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable), return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable),
descriptor->file(), immutable, descriptor->file(), immutable,
IsOwnFile(descriptor, immutable)); IsOwnFile(descriptor, immutable), kotlin);
} }
// Get the Java Class style full name of a message. // Get the Java Class style full name of a message.
std::string ClassNameResolver::GetJavaClassFullName( std::string ClassNameResolver::GetJavaClassFullName(
const std::string& name_without_package, const FileDescriptor* file, const std::string& name_without_package, const FileDescriptor* file,
bool immutable) { bool immutable) {
return GetJavaClassFullName(name_without_package, file, immutable, false);
}
std::string ClassNameResolver::GetJavaClassFullName(
const std::string& name_without_package, const FileDescriptor* file,
bool immutable, bool kotlin) {
std::string result; std::string result;
if (MultipleJavaFiles(file, immutable)) { if (MultipleJavaFiles(file, immutable)) {
result = FileJavaPackage(file, immutable); result = FileJavaPackage(file, immutable);
if (!result.empty()) result += '.'; if (!result.empty()) result += '.';
} else { } else {
result = GetClassName(file, immutable); result = GetClassName(file, immutable, kotlin);
if (!result.empty()) result += '$'; if (!result.empty()) result += '$';
} }
result += StringReplace(name_without_package, ".", "$", true); result += StringReplace(name_without_package, ".", "$", true);
@ -263,7 +304,12 @@ std::string ClassNameResolver::GetJavaClassFullName(
std::string ClassNameResolver::GetExtensionIdentifierName( std::string ClassNameResolver::GetExtensionIdentifierName(
const FieldDescriptor* descriptor, bool immutable) { const FieldDescriptor* descriptor, bool immutable) {
return GetClassName(descriptor->containing_type(), immutable) + "." + return GetExtensionIdentifierName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetExtensionIdentifierName(
const FieldDescriptor* descriptor, bool immutable, bool kotlin) {
return GetClassName(descriptor->containing_type(), immutable, kotlin) + "." +
descriptor->name(); descriptor->name();
} }

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save