Merge tag 'refs/tags/sync-piper' into sync-stage

pull/10005/head
David L. Jones 3 years ago
commit d87d99d931
  1. 9
      java/core/src/main/java/com/google/protobuf/Descriptors.java
  2. 26
      java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java
  3. 124
      java/core/src/main/java/com/google/protobuf/TextFormat.java
  4. 16
      java/core/src/test/java/com/google/protobuf/TextFormatTest.java
  5. 15
      python/google/protobuf/descriptor_pool.py
  6. 2
      python/google/protobuf/internal/descriptor_pool_test.py
  7. 41
      python/google/protobuf/internal/text_format_test.py
  8. 2
      python/google/protobuf/pyext/descriptor.cc
  9. 4
      python/google/protobuf/pyext/message.cc
  10. 18
      python/google/protobuf/text_format.py
  11. 15
      src/Makefile.am
  12. 9
      src/google/protobuf/arenaz_sampler.cc
  13. 6
      src/google/protobuf/arenaz_sampler.h
  14. 14
      src/google/protobuf/compiler/command_line_interface_unittest.cc
  15. 4
      src/google/protobuf/compiler/cpp/message.cc
  16. 63
      src/google/protobuf/compiler/cpp/parse_function_generator.cc
  17. 4
      src/google/protobuf/compiler/cpp/test_bad_identifiers.proto
  18. 21
      src/google/protobuf/compiler/cpp/unittest.inc
  19. 60
      src/google/protobuf/compiler/java/enum_field.cc
  20. 17
      src/google/protobuf/compiler/java/map_field.cc
  21. 32
      src/google/protobuf/compiler/java/map_field_lite.cc
  22. 11
      src/google/protobuf/compiler/java/message.cc
  23. 8
      src/google/protobuf/compiler/plugin.pb.cc
  24. 44
      src/google/protobuf/descriptor.pb.cc
  25. 7
      src/google/protobuf/descriptor.proto
  26. 13
      src/google/protobuf/generated_message_reflection.cc
  27. 18
      src/google/protobuf/generated_message_tctable_decl.h
  28. 41
      src/google/protobuf/generated_message_tctable_impl.h
  29. 229
      src/google/protobuf/generated_message_tctable_lite.cc
  30. 1
      src/google/protobuf/io/coded_stream.h
  31. 1
      src/google/protobuf/io/gzip_stream.h
  32. 1
      src/google/protobuf/io/printer.h
  33. 1
      src/google/protobuf/io/tokenizer.h
  34. 1
      src/google/protobuf/io/zero_copy_stream.h
  35. 1
      src/google/protobuf/io/zero_copy_stream_impl.h
  36. 4
      src/google/protobuf/io/zero_copy_stream_impl_lite.cc
  37. 38
      src/google/protobuf/io/zero_copy_stream_impl_lite.h
  38. 3
      src/google/protobuf/io/zero_copy_stream_unittest.cc
  39. 14
      src/google/protobuf/map_test.inc
  40. 2
      src/google/protobuf/message.h
  41. 1
      src/google/protobuf/message_lite.cc
  42. 4
      src/google/protobuf/message_unittest.inc
  43. 3
      src/google/protobuf/parse_context.cc
  44. 44
      src/google/protobuf/parse_context.h
  45. 52
      src/google/protobuf/stubs/port.h
  46. 14
      src/google/protobuf/test_util2.h
  47. 20
      src/google/protobuf/text_format.cc
  48. 17
      src/google/protobuf/text_format_unittest.cc
  49. 10
      src/google/protobuf/unittest_mset.proto
  50. 2
      src/google/protobuf/util/internal/default_value_objectwriter_test.cc
  51. 8
      src/google/protobuf/util/internal/protostream_objectsource_test.cc
  52. 12
      src/google/protobuf/util/internal/protostream_objectwriter_test.cc
  53. 6
      src/google/protobuf/util/json_util_test.cc
  54. 2
      src/google/protobuf/util/message_differencer_unittest.cc
  55. 75
      src/google/protobuf/util/time_util.cc
  56. 26
      src/google/protobuf/util/time_util.h
  57. 132
      src/google/protobuf/util/time_util_test.cc
  58. 2
      src/google/protobuf/util/type_resolver_util_test.cc

@ -461,21 +461,20 @@ public final class Descriptors {
} }
/** /**
* This method is to be called by generated code only. It is used to update the * This method is to be called by generated code only. It updates the
* FileDescriptorProto associated with the descriptor by parsing it again with the given * FileDescriptorProto associated with the descriptor by parsing it again with the given
* ExtensionRegistry. This is needed to recognize custom options. * ExtensionRegistry. This is needed to recognize custom options.
*/ */
public static void internalUpdateFileDescriptor( public static void internalUpdateFileDescriptor(
final FileDescriptor descriptor, final ExtensionRegistry registry) { FileDescriptor descriptor, ExtensionRegistry registry) {
ByteString bytes = descriptor.proto.toByteString(); ByteString bytes = descriptor.proto.toByteString();
FileDescriptorProto proto;
try { try {
proto = FileDescriptorProto.parseFrom(bytes, registry); FileDescriptorProto proto = FileDescriptorProto.parseFrom(bytes, registry);
descriptor.setProto(proto);
} catch (InvalidProtocolBufferException e) { } catch (InvalidProtocolBufferException e) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Failed to parse protocol buffer descriptor for generated code.", e); "Failed to parse protocol buffer descriptor for generated code.", e);
} }
descriptor.setProto(proto);
} }
/** /**

@ -244,11 +244,16 @@ public abstract class GeneratedMessageLite<
* *
* <p>For use by generated code only. * <p>For use by generated code only.
*/ */
protected abstract Object dynamicMethod(MethodToInvoke method, Object arg0, Object arg1); protected abstract Object dynamicMethod(
MethodToInvoke method,
Object arg0,
Object arg1);
/** Same as {@link #dynamicMethod(MethodToInvoke, Object, Object)} with {@code null} padding. */ /** Same as {@link #dynamicMethod(MethodToInvoke, Object, Object)} with {@code null} padding. */
@CanIgnoreReturnValue @CanIgnoreReturnValue
protected Object dynamicMethod(MethodToInvoke method, Object arg0) { protected Object dynamicMethod(
MethodToInvoke method,
Object arg0) {
return dynamicMethod(method, arg0, null); return dynamicMethod(method, arg0, null);
} }
@ -1245,11 +1250,11 @@ public abstract class GeneratedMessageLite<
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
Object fromFieldSetType(final Object value) { Object fromFieldSetType(Object value) {
if (descriptor.isRepeated()) { if (descriptor.isRepeated()) {
if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
final List result = new ArrayList<>(); List<Object> result = new ArrayList<>();
for (final Object element : (List) value) { for (Object element : (List) value) {
result.add(singularFromFieldSetType(element)); result.add(singularFromFieldSetType(element));
} }
return result; return result;
@ -1261,7 +1266,7 @@ public abstract class GeneratedMessageLite<
} }
} }
Object singularFromFieldSetType(final Object value) { Object singularFromFieldSetType(Object value) {
if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
return descriptor.enumTypeMap.findValueByNumber((Integer) value); return descriptor.enumTypeMap.findValueByNumber((Integer) value);
} else { } else {
@ -1269,12 +1274,11 @@ public abstract class GeneratedMessageLite<
} }
} }
@SuppressWarnings("unchecked") Object toFieldSetType(Object value) {
Object toFieldSetType(final Object value) {
if (descriptor.isRepeated()) { if (descriptor.isRepeated()) {
if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
final List result = new ArrayList<>(); List<Object> result = new ArrayList<>();
for (final Object element : (List) value) { for (Object element : (List) value) {
result.add(singularToFieldSetType(element)); result.add(singularToFieldSetType(element));
} }
return result; return result;
@ -1286,7 +1290,7 @@ public abstract class GeneratedMessageLite<
} }
} }
Object singularToFieldSetType(final Object value) { Object singularToFieldSetType(Object value) {
if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
return ((Internal.EnumLite) value).getNumber(); return ((Internal.EnumLite) value).getNumber();
} else { } else {

@ -116,8 +116,8 @@ public final class TextFormat {
/** /**
* Generates a human readable form of this message, useful for debugging and other purposes, with * Generates a human readable form of this message, useful for debugging and other purposes, with
* no newline characters. This is just a trivial wrapper around * no newline characters. This is just a trivial wrapper around {@link
* {@link TextFormat.Printer#shortDebugString(MessageOrBuilder)}. * TextFormat.Printer#shortDebugString(MessageOrBuilder)}.
*/ */
public static String shortDebugString(final MessageOrBuilder message) { public static String shortDebugString(final MessageOrBuilder message) {
return printer().shortDebugString(message); return printer().shortDebugString(message);
@ -459,9 +459,7 @@ public final class TextFormat {
} }
} }
/** /** An adapter class that can take a {@link MapEntry} and returns its key and entry. */
* An adapter class that can take a {@link MapEntry} and returns its key and entry.
*/
private static class MapEntryAdapter implements Comparable<MapEntryAdapter> { private static class MapEntryAdapter implements Comparable<MapEntryAdapter> {
private Object entry; private Object entry;
@ -953,6 +951,7 @@ public final class TextFormat {
* the next token is parsed. * the next token is parsed.
*/ */
private boolean containsSilentMarkerAfterCurrentToken = false; private boolean containsSilentMarkerAfterCurrentToken = false;
private boolean containsSilentMarkerAfterPrevToken = false; private boolean containsSilentMarkerAfterPrevToken = false;
/** Construct a tokenizer that parses tokens from the given text. */ /** Construct a tokenizer that parses tokens from the given text. */
@ -1378,7 +1377,6 @@ public final class TextFormat {
private ParseException floatParseException(final NumberFormatException e) { private ParseException floatParseException(final NumberFormatException e) {
return parseException("Couldn't parse number: " + e.getMessage()); return parseException("Couldn't parse number: " + e.getMessage());
} }
} }
/** Thrown when parsing an invalid text format message. */ /** Thrown when parsing an invalid text format message. */
@ -1551,7 +1549,7 @@ public final class TextFormat {
* the current token is part of the field value, so the silent marker is indicated by * the current token is part of the field value, so the silent marker is indicated by
* containsSilentMarkerAfterPrevToken. * containsSilentMarkerAfterPrevToken.
*/ */
private void detectSilentMarker(Tokenizer tokenizer) { private void detectSilentMarker(Tokenizer tokenizer, String fieldName) {
} }
/** /**
@ -1628,8 +1626,8 @@ public final class TextFormat {
* unknown field is encountered. If this is set, the parser will only log a warning. Allow * unknown field is encountered. If this is set, the parser will only log a warning. Allow
* unknown fields will also allow unknown extensions. * unknown fields will also allow unknown extensions.
* *
* <p>Use of this parameter is discouraged which may hide some errors (e.g. * <p>Use of this parameter is discouraged which may hide some errors (e.g. spelling error on
* spelling error on field name). * field name).
*/ */
public Builder setAllowUnknownFields(boolean allowUnknownFields) { public Builder setAllowUnknownFields(boolean allowUnknownFields) {
this.allowUnknownFields = allowUnknownFields; this.allowUnknownFields = allowUnknownFields;
@ -1637,10 +1635,9 @@ public final class TextFormat {
} }
/** /**
* Set whether this parser will allow unknown extensions. By default, an * Set whether this parser will allow unknown extensions. By default, an exception is thrown
* exception is thrown if unknown extension is encountered. If this is set true, * if unknown extension is encountered. If this is set true, the parser will only log a
* the parser will only log a warning. Allow unknown extensions does not mean * warning. Allow unknown extensions does not mean allow normal unknown fields.
* allow normal unknown fields.
*/ */
public Builder setAllowUnknownExtensions(boolean allowUnknownExtensions) { public Builder setAllowUnknownExtensions(boolean allowUnknownExtensions) {
this.allowUnknownExtensions = allowUnknownExtensions; this.allowUnknownExtensions = allowUnknownExtensions;
@ -1725,7 +1722,8 @@ public final class TextFormat {
static final class UnknownField { static final class UnknownField {
static enum Type { static enum Type {
FIELD, EXTENSION; FIELD,
EXTENSION;
} }
final String message; final String message;
@ -1786,7 +1784,6 @@ public final class TextFormat {
throws ParseException { throws ParseException {
final Tokenizer tokenizer = new Tokenizer(input); final Tokenizer tokenizer = new Tokenizer(input);
MessageReflection.BuilderAdapter target = new MessageReflection.BuilderAdapter(builder); MessageReflection.BuilderAdapter target = new MessageReflection.BuilderAdapter(builder);
List<UnknownField> unknownFields = new ArrayList<UnknownField>(); List<UnknownField> unknownFields = new ArrayList<UnknownField>();
while (!tokenizer.atEnd()) { while (!tokenizer.atEnd()) {
@ -1803,12 +1800,7 @@ public final class TextFormat {
final MessageReflection.MergeTarget target, final MessageReflection.MergeTarget target,
List<UnknownField> unknownFields) List<UnknownField> unknownFields)
throws ParseException { throws ParseException {
mergeField( mergeField(tokenizer, extensionRegistry, target, parseInfoTreeBuilder, unknownFields);
tokenizer,
extensionRegistry,
target,
parseInfoTreeBuilder,
unknownFields);
} }
/** Parse a single field from {@code tokenizer} and merge it into {@code target}. */ /** Parse a single field from {@code tokenizer} and merge it into {@code target}. */
@ -1820,26 +1812,28 @@ public final class TextFormat {
List<UnknownField> unknownFields) List<UnknownField> unknownFields)
throws ParseException { throws ParseException {
FieldDescriptor field = null; FieldDescriptor field = null;
String name;
int startLine = tokenizer.getLine(); int startLine = tokenizer.getLine();
int startColumn = tokenizer.getColumn(); int startColumn = tokenizer.getColumn();
final Descriptor type = target.getDescriptorForType(); final Descriptor type = target.getDescriptorForType();
ExtensionRegistry.ExtensionInfo extension = null; ExtensionRegistry.ExtensionInfo extension = null;
if ("google.protobuf.Any".equals(type.getFullName()) && tokenizer.tryConsume("[")) { if ("google.protobuf.Any".equals(type.getFullName()) && tokenizer.tryConsume("[")) {
mergeAnyFieldValue(tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, mergeAnyFieldValue(
type); tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, type);
return; return;
} }
if (tokenizer.tryConsume("[")) { if (tokenizer.tryConsume("[")) {
// An extension. // An extension.
final StringBuilder name = new StringBuilder(tokenizer.consumeIdentifier()); StringBuilder nameBuilder = new StringBuilder(tokenizer.consumeIdentifier());
while (tokenizer.tryConsume(".")) { while (tokenizer.tryConsume(".")) {
name.append('.'); nameBuilder.append('.');
name.append(tokenizer.consumeIdentifier()); nameBuilder.append(tokenizer.consumeIdentifier());
} }
name = nameBuilder.toString();
extension = target.findExtensionByName(extensionRegistry, name.toString()); extension = target.findExtensionByName(extensionRegistry, name);
if (extension == null) { if (extension == null) {
String message = String message =
@ -1866,7 +1860,7 @@ public final class TextFormat {
tokenizer.consume("]"); tokenizer.consume("]");
} else { } else {
final String name = tokenizer.consumeIdentifier(); name = tokenizer.consumeIdentifier();
field = type.findFieldByName(name); field = type.findFieldByName(name);
// Group names are expected to be capitalized as they appear in the // Group names are expected to be capitalized as they appear in the
@ -1890,13 +1884,14 @@ public final class TextFormat {
} }
if (field == null) { if (field == null) {
String message = (tokenizer.getPreviousLine() + 1) String message =
+ ":" (tokenizer.getPreviousLine() + 1)
+ (tokenizer.getPreviousColumn() + 1) + ":"
+ ":\t" + (tokenizer.getPreviousColumn() + 1)
+ type.getFullName() + ":\t"
+ "." + type.getFullName()
+ name; + "."
+ name;
unknownFields.add(new UnknownField(message, UnknownField.Type.FIELD)); unknownFields.add(new UnknownField(message, UnknownField.Type.FIELD));
} }
} }
@ -1909,7 +1904,7 @@ public final class TextFormat {
// start with "{" or "<" which indicates the beginning of a message body. // start with "{" or "<" which indicates the beginning of a message body.
// If there is no ":" or there is a "{" or "<" after ":", this field has // If there is no ":" or there is a "{" or "<" after ":", this field has
// to be a message or the input is ill-formed. // to be a message or the input is ill-formed.
detectSilentMarker(tokenizer); detectSilentMarker(tokenizer, name);
if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("{") && !tokenizer.lookingAt("<")) { if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("{") && !tokenizer.lookingAt("<")) {
skipFieldValue(tokenizer); skipFieldValue(tokenizer);
} else { } else {
@ -1920,7 +1915,7 @@ public final class TextFormat {
// Handle potential ':'. // Handle potential ':'.
if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
detectSilentMarker(tokenizer); detectSilentMarker(tokenizer, field.getFullName());
tokenizer.tryConsume(":"); // optional tokenizer.tryConsume(":"); // optional
if (parseTreeBuilder != null) { if (parseTreeBuilder != null) {
TextFormatParseInfoTree.Builder childParseTreeBuilder = TextFormatParseInfoTree.Builder childParseTreeBuilder =
@ -1944,7 +1939,7 @@ public final class TextFormat {
unknownFields); unknownFields);
} }
} else { } else {
detectSilentMarker(tokenizer); detectSilentMarker(tokenizer, field.getFullName());
tokenizer.consume(":"); // required tokenizer.consume(":"); // required
consumeFieldValues( consumeFieldValues(
tokenizer, tokenizer,
@ -1967,6 +1962,29 @@ public final class TextFormat {
} }
} }
private String consumeFullTypeName(Tokenizer tokenizer) throws ParseException {
// If there is not a leading `[`, this is just a type name.
if (!tokenizer.tryConsume("[")) {
return tokenizer.consumeIdentifier();
}
// Otherwise, this is an extension or google.protobuf.Any type URL: we consume proto path
// elements until we've addressed the type.
String name = tokenizer.consumeIdentifier();
while (tokenizer.tryConsume(".")) {
name += "." + tokenizer.consumeIdentifier();
}
if (tokenizer.tryConsume("/")) {
name += "/" + tokenizer.consumeIdentifier();
while (tokenizer.tryConsume(".")) {
name += "." + tokenizer.consumeIdentifier();
}
}
tokenizer.consume("]");
return name;
}
/** /**
* Parse a one or more field values from {@code tokenizer} and merge it into {@code builder}. * Parse a one or more field values from {@code tokenizer} and merge it into {@code builder}.
*/ */
@ -2058,8 +2076,13 @@ public final class TextFormat {
// (java_proto_library for any_java_proto depends on the protobuf_impl). // (java_proto_library for any_java_proto depends on the protobuf_impl).
Message anyBuilder = DynamicMessage.getDefaultInstance(field.getMessageType()); Message anyBuilder = DynamicMessage.getDefaultInstance(field.getMessageType());
MessageReflection.MergeTarget anyField = target.newMergeTargetForField(field, anyBuilder); MessageReflection.MergeTarget anyField = target.newMergeTargetForField(field, anyBuilder);
mergeAnyFieldValue(tokenizer, extensionRegistry, anyField, parseTreeBuilder, mergeAnyFieldValue(
unknownFields, field.getMessageType()); tokenizer,
extensionRegistry,
anyField,
parseTreeBuilder,
unknownFields,
field.getMessageType());
value = anyField.finish(); value = anyField.finish();
tokenizer.consume(endToken); tokenizer.consume(endToken);
} else { } else {
@ -2206,7 +2229,7 @@ public final class TextFormat {
throw tokenizer.parseExceptionPreviousToken("Expected a valid type URL."); throw tokenizer.parseExceptionPreviousToken("Expected a valid type URL.");
} }
} }
detectSilentMarker(tokenizer); detectSilentMarker(tokenizer, typeUrlBuilder.toString());
tokenizer.tryConsume(":"); tokenizer.tryConsume(":");
final String anyEndToken; final String anyEndToken;
if (tokenizer.tryConsume("<")) { if (tokenizer.tryConsume("<")) {
@ -2244,15 +2267,7 @@ public final class TextFormat {
/** Skips the next field including the field's name and value. */ /** Skips the next field including the field's name and value. */
private void skipField(Tokenizer tokenizer) throws ParseException { private void skipField(Tokenizer tokenizer) throws ParseException {
if (tokenizer.tryConsume("[")) { String name = consumeFullTypeName(tokenizer);
// Extension name.
do {
tokenizer.consumeIdentifier();
} while (tokenizer.tryConsume("."));
tokenizer.consume("]");
} else {
tokenizer.consumeIdentifier();
}
// Try to guess the type of this field. // Try to guess the type of this field.
// If this field is not a message, there should be a ":" between the // If this field is not a message, there should be a ":" between the
@ -2260,7 +2275,7 @@ public final class TextFormat {
// start with "{" or "<" which indicates the beginning of a message body. // start with "{" or "<" which indicates the beginning of a message body.
// If there is no ":" or there is a "{" or "<" after ":", this field has // If there is no ":" or there is a "{" or "<" after ":", this field has
// to be a message or the input is ill-formed. // to be a message or the input is ill-formed.
detectSilentMarker(tokenizer); detectSilentMarker(tokenizer, name);
if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("<") && !tokenizer.lookingAt("{")) { if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("<") && !tokenizer.lookingAt("{")) {
skipFieldValue(tokenizer); skipFieldValue(tokenizer);
} else { } else {
@ -2469,9 +2484,10 @@ public final class TextFormat {
} }
Character.UnicodeBlock unicodeBlock = Character.UnicodeBlock.of(codepoint); Character.UnicodeBlock unicodeBlock = Character.UnicodeBlock.of(codepoint);
if (unicodeBlock != null if (unicodeBlock != null
&& (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES) && (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES)
|| unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES) || unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES)
|| unicodeBlock.equals(Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES))) { || unicodeBlock.equals(
Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES))) {
throw new InvalidEscapeSequenceException( throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U" "Invalid escape sequence: '\\U"
+ input.substring(i, i + 8).toStringUtf8() + input.substring(i, i + 8).toStringUtf8()

@ -67,9 +67,7 @@ import org.junit.function.ThrowingRunnable;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.junit.runners.JUnit4; import org.junit.runners.JUnit4;
/** /** Test case for {@link TextFormat}. */
* Test case for {@link TextFormat}.
*/
@RunWith(JUnit4.class) @RunWith(JUnit4.class)
public class TextFormatTest { public class TextFormatTest {
@ -1450,6 +1448,18 @@ public class TextFormatTest {
+ "unknown_field3: 3\n"); + "unknown_field3: 3\n");
} }
@Test
public void testParseUnknownExtensionWithAnyMessage() throws Exception {
assertParseSuccessWithUnknownExtensions(
"[unknown_extension]: { "
+ " any_value { "
+ " [type.googleapis.com/protobuf_unittest.OneString] { "
+ " data: 123 "
+ " } "
+ " } "
+ "}");
}
// See additional coverage in testOneofOverwriteForbidden and testMapOverwriteForbidden. // See additional coverage in testOneofOverwriteForbidden and testMapOverwriteForbidden.
@Test @Test
public void testParseNonRepeatedFields() throws Exception { public void testParseNonRepeatedFields() throws Exception {

@ -144,9 +144,6 @@ class DescriptorPool(object):
self._service_descriptors = {} self._service_descriptors = {}
self._file_descriptors = {} self._file_descriptors = {}
self._toplevel_extensions = {} self._toplevel_extensions = {}
# TODO(jieluo): Remove _file_desc_by_toplevel_extension after
# maybe year 2020 for compatibility issue (with 3.4.1 only).
self._file_desc_by_toplevel_extension = {}
self._top_enum_values = {} self._top_enum_values = {}
# We store extensions in two two-level mappings: The first key is the # We store extensions in two two-level mappings: The first key is the
# descriptor of the message being extended, the second key is the extension # descriptor of the message being extended, the second key is the extension
@ -331,6 +328,8 @@ class DescriptorPool(object):
raise TypeError('Expected an extension descriptor.') raise TypeError('Expected an extension descriptor.')
if extension.extension_scope is None: if extension.extension_scope is None:
self._CheckConflictRegister(
extension, extension.full_name, extension.file.name)
self._toplevel_extensions[extension.full_name] = extension self._toplevel_extensions[extension.full_name] = extension
try: try:
@ -372,12 +371,6 @@ class DescriptorPool(object):
""" """
self._AddFileDescriptor(file_desc) self._AddFileDescriptor(file_desc)
# TODO(jieluo): This is a temporary solution for FieldDescriptor.file.
# FieldDescriptor.file is added in code gen. Remove this solution after
# maybe 2020 for compatibility reason (with 3.4.1 only).
for extension in file_desc.extensions_by_name.values():
self._file_desc_by_toplevel_extension[
extension.full_name] = file_desc
def _AddFileDescriptor(self, file_desc): def _AddFileDescriptor(self, file_desc):
"""Adds a FileDescriptor to the pool, non-recursively. """Adds a FileDescriptor to the pool, non-recursively.
@ -483,7 +476,7 @@ class DescriptorPool(object):
pass pass
try: try:
return self._file_desc_by_toplevel_extension[symbol] return self._toplevel_extensions[symbol].file
except KeyError: except KeyError:
pass pass
@ -792,8 +785,6 @@ class DescriptorPool(object):
file_descriptor.package, scope) file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = ( file_descriptor.extensions_by_name[extension_desc.name] = (
extension_desc) extension_desc)
self._file_desc_by_toplevel_extension[extension_desc.full_name] = (
file_descriptor)
for desc_proto in file_proto.message_type: for desc_proto in file_proto.message_type:
self._SetAllFieldTypes(file_proto.package, desc_proto, scope) self._SetAllFieldTypes(file_proto.package, desc_proto, scope)

@ -33,7 +33,6 @@
__author__ = 'matthewtoia@google.com (Matt Toia)' __author__ = 'matthewtoia@google.com (Matt Toia)'
import copy import copy
import os
import unittest import unittest
import warnings import warnings
@ -415,7 +414,6 @@ class DescriptorPoolTestBase(object):
field = file_json.message_types_by_name['class'].fields_by_name['int_field'] field = file_json.message_types_by_name['class'].fields_by_name['int_field']
self.assertEqual(field.json_name, 'json_int') self.assertEqual(field.json_name, 'json_int')
def testEnumDefaultValue(self): def testEnumDefaultValue(self):
"""Test the default value of enums which don't start at zero.""" """Test the default value of enums which don't start at zero."""
def _CheckDefaultValue(file_descriptor): def _CheckDefaultValue(file_descriptor):

@ -1598,6 +1598,47 @@ class Proto2Tests(TextFormatBase):
self.assertEqual(23, message.message_set.Extensions[ext1].i) self.assertEqual(23, message.message_set.Extensions[ext1].i)
self.assertEqual('foo', message.message_set.Extensions[ext2].str) self.assertEqual('foo', message.message_set.Extensions[ext2].str)
# Handle Any messages inside unknown extensions.
message = any_test_pb2.TestAny()
text = ('any_value {\n'
' [type.googleapis.com/google.protobuf.internal.TestAny] {\n'
' [unknown_extension] {\n'
' str: "string"\n'
' any_value {\n'
' [type.googleapis.com/protobuf_unittest.OneString] {\n'
' data: "string"\n'
' }\n'
' }\n'
' }\n'
' }\n'
'}\n'
'int32_value: 123')
text_format.Parse(text, message, allow_unknown_extension=True)
self.assertEqual(123, message.int32_value)
# Fail if invalid Any message type url inside unknown extensions.
message = any_test_pb2.TestAny()
text = ('any_value {\n'
' [type.googleapis.com.invalid/google.protobuf.internal.TestAny] {\n'
' [unknown_extension] {\n'
' str: "string"\n'
' any_value {\n'
' [type.googleapis.com/protobuf_unittest.OneString] {\n'
' data: "string"\n'
' }\n'
' }\n'
' }\n'
' }\n'
'}\n'
'int32_value: 123')
self.assertRaisesRegex(
text_format.ParseError,
'[type.googleapis.com.invalid/google.protobuf.internal.TestAny]',
text_format.Parse,
text,
message,
allow_unknown_extension=True)
def testParseBadIdentifier(self): def testParseBadIdentifier(self):
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
text = ('optional_nested_message { "bb": 1 }') text = ('optional_nested_message { "bb": 1 }')

@ -40,7 +40,6 @@
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/dynamic_message.h> #include <google/protobuf/dynamic_message.h>
#include <google/protobuf/pyext/descriptor_containers.h> #include <google/protobuf/pyext/descriptor_containers.h>
@ -48,6 +47,7 @@
#include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/message_factory.h> #include <google/protobuf/pyext/message_factory.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/stubs/hash.h> #include <google/protobuf/stubs/hash.h>
#define PyString_AsStringAndSize(ob, charpp, sizep) \ #define PyString_AsStringAndSize(ob, charpp, sizep) \

@ -51,8 +51,6 @@
#endif #endif
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/logging.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor.h>
#include <google/protobuf/message.h> #include <google/protobuf/message.h>
@ -71,7 +69,9 @@
#include <google/protobuf/pyext/unknown_field_set.h> #include <google/protobuf/pyext/unknown_field_set.h>
#include <google/protobuf/pyext/unknown_fields.h> #include <google/protobuf/pyext/unknown_fields.h>
#include <google/protobuf/util/message_differencer.h> #include <google/protobuf/util/message_differencer.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/strtod.h> #include <google/protobuf/io/strtod.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/stubs/map_util.h> #include <google/protobuf/stubs/map_util.h>
// clang-format off // clang-format off

@ -856,9 +856,12 @@ class _Parser(object):
ParseError: On text parsing problems. ParseError: On text parsing problems.
""" """
# Tokenize expects native str lines. # Tokenize expects native str lines.
str_lines = ( try:
line if isinstance(line, str) else line.decode('utf-8') str_lines = (
for line in lines) line if isinstance(line, str) else line.decode('utf-8')
for line in lines)
except UnicodeDecodeError as e:
raise self._StringParseError(e)
tokenizer = Tokenizer(str_lines) tokenizer = Tokenizer(str_lines)
while not tokenizer.AtEnd(): while not tokenizer.AtEnd():
self._MergeField(tokenizer, message) self._MergeField(tokenizer, message)
@ -1190,10 +1193,17 @@ def _SkipField(tokenizer):
tokenizer: A tokenizer to parse the field name and values. tokenizer: A tokenizer to parse the field name and values.
""" """
if tokenizer.TryConsume('['): if tokenizer.TryConsume('['):
# Consume extension name. # Consume extension or google.protobuf.Any type URL
tokenizer.ConsumeIdentifier() tokenizer.ConsumeIdentifier()
num_identifiers = 1
while tokenizer.TryConsume('.'): while tokenizer.TryConsume('.'):
tokenizer.ConsumeIdentifier() tokenizer.ConsumeIdentifier()
num_identifiers += 1
# This is possibly a type URL for an Any message.
if num_identifiers == 3 and tokenizer.TryConsume('/'):
tokenizer.ConsumeIdentifier()
while tokenizer.TryConsume('.'):
tokenizer.ConsumeIdentifier()
tokenizer.Consume(']') tokenizer.Consume(']')
else: else:
tokenizer.ConsumeIdentifierOrNumber() tokenizer.ConsumeIdentifierOrNumber()

@ -540,18 +540,8 @@ protoc_inputs = \
EXTRA_DIST = \ EXTRA_DIST = \
$(protoc_inputs) \ $(protoc_inputs) \
BUILD.bazel \
README.md \ README.md \
google/protobuf/BUILD.bazel \
google/protobuf/compiler/BUILD.bazel \
google/protobuf/compiler/cpp/BUILD.bazel \
google/protobuf/compiler/csharp/BUILD.bazel \
google/protobuf/compiler/java/BUILD.bazel \
google/protobuf/compiler/objectivec/BUILD.bazel \
google/protobuf/compiler/package_info.h \ google/protobuf/compiler/package_info.h \
google/protobuf/compiler/php/BUILD.bazel \
google/protobuf/compiler/python/BUILD.bazel \
google/protobuf/compiler/ruby/BUILD.bazel \
google/protobuf/compiler/ruby/ruby_generated_code.proto \ google/protobuf/compiler/ruby/ruby_generated_code.proto \
google/protobuf/compiler/ruby/ruby_generated_code_pb.rb \ google/protobuf/compiler/ruby/ruby_generated_code_pb.rb \
google/protobuf/compiler/ruby/ruby_generated_code_proto2.proto \ google/protobuf/compiler/ruby/ruby_generated_code_proto2.proto \
@ -564,12 +554,10 @@ EXTRA_DIST = \
google/protobuf/compiler/ruby/ruby_generated_pkg_implicit.proto \ google/protobuf/compiler/ruby/ruby_generated_pkg_implicit.proto \
google/protobuf/compiler/ruby/ruby_generated_pkg_implicit_pb.rb \ google/protobuf/compiler/ruby/ruby_generated_pkg_implicit_pb.rb \
google/protobuf/compiler/zip_output_unittest.sh \ google/protobuf/compiler/zip_output_unittest.sh \
google/protobuf/io/BUILD.bazel \
google/protobuf/io/gzip_stream.h \ google/protobuf/io/gzip_stream.h \
google/protobuf/io/gzip_stream_unittest.sh \ google/protobuf/io/gzip_stream_unittest.sh \
google/protobuf/io/package_info.h \ google/protobuf/io/package_info.h \
google/protobuf/package_info.h \ google/protobuf/package_info.h \
google/protobuf/stubs/BUILD.bazel \
google/protobuf/test_messages_proto2.proto \ google/protobuf/test_messages_proto2.proto \
google/protobuf/test_messages_proto3.proto \ google/protobuf/test_messages_proto3.proto \
google/protobuf/testdata/bad_utf8_string \ google/protobuf/testdata/bad_utf8_string \
@ -585,9 +573,6 @@ EXTRA_DIST = \
google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt \ google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt \
google/protobuf/testdata/text_format_unittest_extensions_data.txt \ google/protobuf/testdata/text_format_unittest_extensions_data.txt \
google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt \ google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt \
google/protobuf/testing/BUILD.bazel \
google/protobuf/util/BUILD.bazel \
google/protobuf/util/internal/BUILD.bazel \
google/protobuf/util/package_info.h \ google/protobuf/util/package_info.h \
libprotobuf-lite.map \ libprotobuf-lite.map \
libprotobuf.map \ libprotobuf.map \

@ -97,15 +97,12 @@ void RecordResetSlow(ThreadSafeArenaStats* info) {
void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested, void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested,
size_t allocated, size_t wasted) { size_t allocated, size_t wasted) {
info->num_allocations.fetch_add(1, std::memory_order_relaxed);
info->bytes_requested.fetch_add(requested, std::memory_order_relaxed); info->bytes_requested.fetch_add(requested, std::memory_order_relaxed);
info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
info->num_allocations.fetch_add(1, std::memory_order_relaxed); const uint64_t tid = 1ULL << (GetCachedTID() % 63);
const uint64_t tid = (1ULL << (GetCachedTID() % 63)); info->thread_ids.fetch_or(tid, std::memory_order_relaxed);
const uint64_t thread_ids = info->thread_ids.load(std::memory_order_relaxed);
if (!(thread_ids & tid)) {
info->thread_ids.store(thread_ids | tid, std::memory_order_relaxed);
}
} }
ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {

@ -70,8 +70,10 @@ struct ThreadSafeArenaStats
std::atomic<size_t> bytes_wasted; std::atomic<size_t> bytes_wasted;
// Records the largest size an arena ever had. Maintained across resets. // Records the largest size an arena ever had. Maintained across resets.
std::atomic<size_t> max_bytes_allocated; std::atomic<size_t> max_bytes_allocated;
// Bit i when set to 1 indicates that a thread with tid % 63 = i accessed the // Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed
// underlying arena. The field is maintained across resets. // the underlying arena. We use `% 63` as a rudimentary hash to ensure some
// bit mixing for thread-ids; `% 64` would only grab the low bits and might
// create sampling artifacts. Maintained across resets.
std::atomic<uint64_t> thread_ids; std::atomic<uint64_t> thread_ids;
// All of the fields below are set by `PrepareForSampling`, they must not // All of the fields below are set by `PrepareForSampling`, they must not

@ -2697,7 +2697,7 @@ class EncodeDecodeTest : public testing::TestWithParam<EncodeDecodeTestMode> {
TEST_P(EncodeDecodeTest, Encode) { TEST_P(EncodeDecodeTest, Encode) {
RedirectStdinFromFile(TestUtil::GetTestDataPath( RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/" "third_party/protobuf/"
"testdata/text_format_unittest_data_oneof_implemented.txt")); "testdata/text_format_unittest_data_oneof_implemented.txt"));
std::string args; std::string args;
if (GetParam() != DESCRIPTOR_SET_IN) { if (GetParam() != DESCRIPTOR_SET_IN) {
@ -2706,17 +2706,17 @@ TEST_P(EncodeDecodeTest, Encode) {
} }
EXPECT_TRUE(Run(args + " --encode=protobuf_unittest.TestAllTypes")); EXPECT_TRUE(Run(args + " --encode=protobuf_unittest.TestAllTypes"));
ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath( ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented")); "third_party/protobuf/testdata/golden_message_oneof_implemented"));
} }
TEST_P(EncodeDecodeTest, Decode) { TEST_P(EncodeDecodeTest, Decode) {
RedirectStdinFromFile(TestUtil::GetTestDataPath( RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented")); "third_party/protobuf/testdata/golden_message_oneof_implemented"));
EXPECT_TRUE( EXPECT_TRUE(
Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") +
" --decode=protobuf_unittest.TestAllTypes")); " --decode=protobuf_unittest.TestAllTypes"));
ExpectStdoutMatchesTextFile(TestUtil::GetTestDataPath( ExpectStdoutMatchesTextFile(TestUtil::GetTestDataPath(
"net/proto2/internal/" "third_party/protobuf/"
"testdata/text_format_unittest_data_oneof_implemented.txt")); "testdata/text_format_unittest_data_oneof_implemented.txt"));
} }
@ -2764,7 +2764,7 @@ TEST_P(EncodeDecodeTest, ProtoParseError) {
TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) { TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) {
RedirectStdinFromFile(TestUtil::GetTestDataPath( RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/" "third_party/protobuf/"
"testdata/text_format_unittest_data_oneof_implemented.txt")); "testdata/text_format_unittest_data_oneof_implemented.txt"));
std::string args; std::string args;
if (GetParam() != DESCRIPTOR_SET_IN) { if (GetParam() != DESCRIPTOR_SET_IN) {
@ -2774,12 +2774,12 @@ TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) {
EXPECT_TRUE(Run( EXPECT_TRUE(Run(
args + " --encode=protobuf_unittest.TestAllTypes --deterministic_output")); args + " --encode=protobuf_unittest.TestAllTypes --deterministic_output"));
ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath( ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented")); "third_party/protobuf/testdata/golden_message_oneof_implemented"));
} }
TEST_P(EncodeDecodeTest, DecodeDeterministicOutput) { TEST_P(EncodeDecodeTest, DecodeDeterministicOutput) {
RedirectStdinFromFile(TestUtil::GetTestDataPath( RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented")); "third_party/protobuf/testdata/golden_message_oneof_implemented"));
EXPECT_FALSE( EXPECT_FALSE(
Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") +
" --decode=protobuf_unittest.TestAllTypes --deterministic_output")); " --decode=protobuf_unittest.TestAllTypes --deterministic_output"));

@ -2101,7 +2101,9 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
if (!has_bit_indices_.empty()) { if (!has_bit_indices_.empty()) {
format( format(
"using HasBits = " "using HasBits = "
"decltype(std::declval<$classname$>().$has_bits$);\n"); "decltype(std::declval<$classname$>().$has_bits$);\n"
"static constexpr int32_t kHasBitsOffset =\n"
" 8 * PROTOBUF_FIELD_OFFSET($classname$, _impl_._has_bits_);\n");
} }
for (auto field : FieldRange(descriptor_)) { for (auto field : FieldRange(descriptor_)) {
field_generators_.get(field).GenerateInternalAccessorDeclarations(printer); field_generators_.get(field).GenerateInternalAccessorDeclarations(printer);

@ -47,6 +47,15 @@ namespace {
using google::protobuf::internal::WireFormat; using google::protobuf::internal::WireFormat;
using google::protobuf::internal::WireFormatLite; using google::protobuf::internal::WireFormatLite;
bool UseDirectTcParserTable(const FieldDescriptor* field,
const Options& options) {
auto* m = field->message_type();
return !m->options().message_set_wire_format() &&
m->file()->options().optimize_for() != FileOptions::CODE_SIZE &&
!HasSimpleBaseClass(m, options) && !HasTracker(m, options)
;
}
std::vector<const FieldDescriptor*> GetOrderedFields( std::vector<const FieldDescriptor*> GetOrderedFields(
const Descriptor* descriptor, const Options& options) { const Descriptor* descriptor, const Options& options) {
std::vector<const FieldDescriptor*> ordered_fields; std::vector<const FieldDescriptor*> ordered_fields;
@ -324,10 +333,17 @@ TailCallTableInfo::TailCallTableInfo(
// Lazy fields are handled by the generated fallback function. // Lazy fields are handled by the generated fallback function.
} else { } else {
field_entries.back().aux_idx = aux_entries.size(); field_entries.back().aux_idx = aux_entries.size();
const Descriptor* field_type = field->message_type(); if (UseDirectTcParserTable(field, options)) {
aux_entries.push_back(StrCat( const Descriptor* field_type = field->message_type();
"reinterpret_cast<const ", QualifiedClassName(field_type, options), aux_entries.push_back(
"*>(&", QualifiedDefaultInstanceName(field_type, options), ")")); StrCat("::_pbi::TcParser::GetTable<",
QualifiedClassName(field_type, options), ">()"));
} else {
const Descriptor* field_type = field->message_type();
aux_entries.push_back(
StrCat("::_pbi::FieldAuxDefaultMessage{}, &",
QualifiedDefaultInstanceName(field_type, options)));
}
} }
} else if (field->type() == FieldDescriptor::TYPE_ENUM && } else if (field->type() == FieldDescriptor::TYPE_ENUM &&
!HasPreservingUnknownEnumSemantics(field)) { !HasPreservingUnknownEnumSemantics(field)) {
@ -518,6 +534,9 @@ bool ParseFunctionGenerator::should_generate_tctable() const {
if (options_.tctable_mode == Options::kTCTableNever) { if (options_.tctable_mode == Options::kTCTableNever) {
return false; return false;
} }
if (HasSimpleBaseClass(descriptor_, options_)) {
return false;
}
return true; return true;
} }
@ -547,7 +566,7 @@ void ParseFunctionGenerator::GenerateTailcallFallbackFunction(
if (num_hasbits_ > 0) { if (num_hasbits_ > 0) {
// Sync hasbits // Sync hasbits
format("typed_msg->_impl_._has_bits_[0] = hasbits;\n"); format("typed_msg->_impl_._has_bits_[0] |= hasbits;\n");
} }
format("uint32_t tag = data.tag();\n"); format("uint32_t tag = data.tag();\n");
@ -604,6 +623,7 @@ void ParseFunctionGenerator::GenerateDataDecls(io::Printer* printer) {
} }
auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_);
format( format(
"friend class ::$proto_ns$::internal::TcParser;\n"
"static const ::$proto_ns$::internal::" "static const ::$proto_ns$::internal::"
"TcParseTable<$1$, $2$, $3$, $4$, $5$> _table_;\n", "TcParseTable<$1$, $2$, $3$, $4$, $5$> _table_;\n",
tc_table_info_->table_size_log2, ordered_fields_.size(), tc_table_info_->table_size_log2, ordered_fields_.size(),
@ -761,7 +781,7 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) {
// unknown fields and potentially an extension range. // unknown fields and potentially an extension range.
auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_);
format( format(
"PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n" "PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n"
"const ::_pbi::TcParseTable<$1$, $2$, $3$, $4$, $5$> " "const ::_pbi::TcParseTable<$1$, $2$, $3$, $4$, $5$> "
"$classname$::_table_ = " "$classname$::_table_ = "
"{\n", "{\n",
@ -1012,6 +1032,11 @@ static void FormatFieldKind(Formatter& format,
case FieldDescriptor::TYPE_GROUP: case FieldDescriptor::TYPE_GROUP:
format("Message | ::_fl::kRepGroup"); format("Message | ::_fl::kRepGroup");
if (UseDirectTcParserTable(field, options)) {
format(" | ::_fl::kTvTable");
} else {
format(" | ::_fl::kTvDefault");
}
break; break;
case FieldDescriptor::TYPE_MESSAGE: case FieldDescriptor::TYPE_MESSAGE:
if (field->is_map()) { if (field->is_map()) {
@ -1023,6 +1048,11 @@ static void FormatFieldKind(Formatter& format,
} else if (IsImplicitWeakField(field, options, scc_analyzer)) { } else if (IsImplicitWeakField(field, options, scc_analyzer)) {
format(" | ::_fl::kRepIWeak"); format(" | ::_fl::kRepIWeak");
} }
if (UseDirectTcParserTable(field, options)) {
format(" | ::_fl::kTvTable");
} else {
format(" | ::_fl::kTvDefault");
}
} }
break; break;
} }
@ -1052,11 +1082,22 @@ void ParseFunctionGenerator::GenerateFieldEntries(Formatter& format) {
} else { } else {
const OneofDescriptor* oneof = field->real_containing_oneof(); const OneofDescriptor* oneof = field->real_containing_oneof();
bool cold = ShouldSplit(field, options_); bool cold = ShouldSplit(field, options_);
format("PROTOBUF_FIELD_OFFSET($classname$$1$, $2$), $3$, $4$,\n ", format("PROTOBUF_FIELD_OFFSET($classname$$1$, $2$), ",
cold ? "::Impl_::Split" : "", cold ? "::Impl_::Split" : "",
cold ? FieldName(field) + "_" cold ? FieldName(field) + "_"
: FieldMemberName(field, /*cold=*/false), : FieldMemberName(field, /*cold=*/false));
(oneof ? oneof->index() : entry.hasbit_idx), entry.aux_idx); if (oneof) {
format("$1$, ", oneof->index());
} else if (num_hasbits_ > 0 || IsMapEntryMessage(descriptor_)) {
if (entry.hasbit_idx >= 0) {
format("_Internal::kHasBitsOffset + $1$, ", entry.hasbit_idx);
} else {
format("$1$, ", entry.hasbit_idx);
}
} else {
format("0, ");
}
format("$1$,\n ", entry.aux_idx);
FormatFieldKind(format, entry, options_, scc_analyzer_); FormatFieldKind(format, entry, options_, scc_analyzer_);
} }
format("},\n"); format("},\n");
@ -1691,10 +1732,10 @@ std::string FieldParseFunctionName(
break; break;
case FieldDescriptor::TYPE_MESSAGE: case FieldDescriptor::TYPE_MESSAGE:
name.append("M"); name.append(UseDirectTcParserTable(field, options) ? "Mt" : "Md");
break; break;
case FieldDescriptor::TYPE_GROUP: case FieldDescriptor::TYPE_GROUP:
name.append("G"); name.append(UseDirectTcParserTable(field, options) ? "Gt" : "Gd");
break; break;
default: default:

@ -61,7 +61,9 @@ message TestConflictingSymbolNames {
optional int32 total_size = 6; optional int32 total_size = 6;
optional int32 tag = 7; optional int32 tag = 7;
enum TestEnum { FOO = 0; } enum TestEnum {
FOO = 0;
}
message Data1 { message Data1 {
repeated int32 data = 1; repeated int32 data = 1;
} }

@ -56,26 +56,25 @@
// visual studio to compile (report internal errors). // visual studio to compile (report internal errors).
#include <google/protobuf/unittest_enormous_descriptor.pb.h> #include <google/protobuf/unittest_enormous_descriptor.pb.h>
#endif #endif
#include <google/protobuf/stubs/callback.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/compiler/cpp/helpers.h> #include <google/protobuf/compiler/cpp/helpers.h>
#include <google/protobuf/compiler/cpp/test_bad_identifiers.pb.h>
#include <google/protobuf/unittest_no_generic_services.pb.h> #include <google/protobuf/unittest_no_generic_services.pb.h>
#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/casts.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/arena.h> #include <google/protobuf/arena.h>
#include <google/protobuf/compiler/cpp/test_bad_identifiers.pb.h>
#include <google/protobuf/compiler/importer.h>
#include <google/protobuf/compiler/scc.h>
#include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h> #include <google/protobuf/dynamic_message.h>
#include <google/protobuf/compiler/scc.h>
#include <google/protobuf/compiler/importer.h>
#include <google/protobuf/io/coded_stream.h> #include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/test_util2.h> #include <google/protobuf/test_util2.h>
#include <google/protobuf/stubs/callback.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/casts.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/stubs/stl_util.h> #include <google/protobuf/stubs/stl_util.h>
// Must be included last. // Must be included last.

@ -42,7 +42,6 @@
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/printer.h> #include <google/protobuf/io/printer.h>
#include <google/protobuf/wire_format.h> #include <google/protobuf/wire_format.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/compiler/java/context.h> #include <google/protobuf/compiler/java/context.h>
#include <google/protobuf/compiler/java/doc_comment.h> #include <google/protobuf/compiler/java/doc_comment.h>
#include <google/protobuf/compiler/java/helpers.h> #include <google/protobuf/compiler/java/helpers.h>
@ -85,13 +84,6 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
? "@kotlin.Deprecated(message = \"Field " + (*variables)["name"] + ? "@kotlin.Deprecated(message = \"Field " + (*variables)["name"] +
" is deprecated\") " " is deprecated\") "
: ""; : "";
(*variables)["on_changed"] = "onChanged();";
// Use deprecated valueOf() method to be compatible with old generated code
// for v2.5.0/v2.6.1.
// TODO(xiaofeng): Use "forNumber" when we no longer support compatibility
// with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations.
(*variables)["for_number"] = "valueOf";
if (HasHasbit(descriptor)) { if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit. // For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
@ -204,8 +196,7 @@ void ImmutableEnumFieldGenerator::GenerateMembers(io::Printer* printer) const {
printer->Print(variables_, printer->Print(variables_,
"@java.lang.Override $deprecation$public $type$ " "@java.lang.Override $deprecation$public $type$ "
"${$get$capitalized_name$$}$() {\n" "${$get$capitalized_name$$}$() {\n"
" @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.forNumber($name$_);\n"
" $type$ result = $type$.$for_number$($name$_);\n"
" return result == null ? $unknown$ : result;\n" " return result == null ? $unknown$ : result;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -238,7 +229,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers(
"${$set$capitalized_name$Value$}$(int value) {\n" "${$set$capitalized_name$Value$}$(int value) {\n"
" $set_has_field_bit_builder$\n" " $set_has_field_bit_builder$\n"
" $name$_ = value;\n" " $name$_ = value;\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -247,8 +238,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers(
printer->Print(variables_, printer->Print(variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
" @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.forNumber($name$_);\n"
" $type$ result = $type$.$for_number$($name$_);\n"
" return result == null ? $unknown$ : result;\n" " return result == null ? $unknown$ : result;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -262,7 +252,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers(
" }\n" " }\n"
" $set_has_field_bit_builder$\n" " $set_has_field_bit_builder$\n"
" $name$_ = value.getNumber();\n" " $name$_ = value.getNumber();\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -273,7 +263,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers(
"$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n"
" $clear_has_field_bit_builder$\n" " $clear_has_field_bit_builder$\n"
" $name$_ = $default_number$;\n" " $name$_ = $default_number$;\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -364,8 +354,7 @@ void ImmutableEnumFieldGenerator::GenerateParsingCode(
} else { } else {
printer->Print(variables_, printer->Print(variables_,
"int rawValue = input.readEnum();\n" "int rawValue = input.readEnum();\n"
" @SuppressWarnings(\"deprecation\")\n" "$type$ value = $type$.forNumber(rawValue);\n"
"$type$ value = $type$.$for_number$(rawValue);\n"
"if (value == null) {\n" "if (value == null) {\n"
" unknownFields.mergeVarintField($number$, rawValue);\n" " unknownFields.mergeVarintField($number$, rawValue);\n"
"} else {\n" "} else {\n"
@ -453,8 +442,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateMembers(
printer->Print(variables_, printer->Print(variables_,
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
" if ($has_oneof_case_message$) {\n" " if ($has_oneof_case_message$) {\n"
" @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.forNumber(\n"
" $type$ result = $type$.$for_number$(\n"
" (java.lang.Integer) $oneof_name$_);\n" " (java.lang.Integer) $oneof_name$_);\n"
" return result == null ? $unknown$ : result;\n" " return result == null ? $unknown$ : result;\n"
" }\n" " }\n"
@ -493,7 +481,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers(
"${$set$capitalized_name$Value$}$(int value) {\n" "${$set$capitalized_name$Value$}$(int value) {\n"
" $set_oneof_case_message$;\n" " $set_oneof_case_message$;\n"
" $oneof_name$_ = value;\n" " $oneof_name$_ = value;\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -503,8 +491,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers(
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n"
" if ($has_oneof_case_message$) {\n" " if ($has_oneof_case_message$) {\n"
" @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.forNumber(\n"
" $type$ result = $type$.$for_number$(\n"
" (java.lang.Integer) $oneof_name$_);\n" " (java.lang.Integer) $oneof_name$_);\n"
" return result == null ? $unknown$ : result;\n" " return result == null ? $unknown$ : result;\n"
" }\n" " }\n"
@ -521,7 +508,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers(
" }\n" " }\n"
" $set_oneof_case_message$;\n" " $set_oneof_case_message$;\n"
" $oneof_name$_ = value.getNumber();\n" " $oneof_name$_ = value.getNumber();\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -533,7 +520,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers(
" if ($has_oneof_case_message$) {\n" " if ($has_oneof_case_message$) {\n"
" $clear_oneof_case_message$;\n" " $clear_oneof_case_message$;\n"
" $oneof_name$_ = null;\n" " $oneof_name$_ = null;\n"
" $on_changed$\n" " onChanged();\n"
" }\n" " }\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
@ -570,8 +557,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateParsingCode(
} else { } else {
printer->Print(variables_, printer->Print(variables_,
"int rawValue = input.readEnum();\n" "int rawValue = input.readEnum();\n"
"@SuppressWarnings(\"deprecation\")\n" "$type$ value = $type$.forNumber(rawValue);\n"
"$type$ value = $type$.$for_number$(rawValue);\n"
"if (value == null) {\n" "if (value == null) {\n"
" unknownFields.mergeVarintField($number$, rawValue);\n" " unknownFields.mergeVarintField($number$, rawValue);\n"
"} else {\n" "} else {\n"
@ -685,8 +671,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers(
" new com.google.protobuf.Internal.ListAdapter.Converter<\n" " new com.google.protobuf.Internal.ListAdapter.Converter<\n"
" java.lang.Integer, $type$>() {\n" " java.lang.Integer, $type$>() {\n"
" public $type$ convert(java.lang.Integer from) {\n" " public $type$ convert(java.lang.Integer from) {\n"
" @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.forNumber(from);\n"
" $type$ result = $type$.$for_number$(from);\n"
" return result == null ? $unknown$ : result;\n" " return result == null ? $unknown$ : result;\n"
" }\n" " }\n"
" };\n"); " };\n");
@ -802,7 +787,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
" }\n" " }\n"
" ensure$capitalized_name$IsMutable();\n" " ensure$capitalized_name$IsMutable();\n"
" $name$_.set(index, value.getNumber());\n" " $name$_.set(index, value.getNumber());\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -816,7 +801,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
" }\n" " }\n"
" ensure$capitalized_name$IsMutable();\n" " ensure$capitalized_name$IsMutable();\n"
" $name$_.add(value.getNumber());\n" " $name$_.add(value.getNumber());\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -829,7 +814,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
" for ($type$ value : values) {\n" " for ($type$ value : values) {\n"
" $name$_.add(value.getNumber());\n" " $name$_.add(value.getNumber());\n"
" }\n" " }\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -840,7 +825,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
"$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n"
" $name$_ = java.util.Collections.emptyList();\n" " $name$_ = java.util.Collections.emptyList();\n"
" $clear_mutable_bit_builder$;\n" " $clear_mutable_bit_builder$;\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -870,7 +855,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
" int index, int value) {\n" " int index, int value) {\n"
" ensure$capitalized_name$IsMutable();\n" " ensure$capitalized_name$IsMutable();\n"
" $name$_.set(index, value);\n" " $name$_.set(index, value);\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -881,7 +866,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
"${$add$capitalized_name$Value$}$(int value) {\n" "${$add$capitalized_name$Value$}$(int value) {\n"
" ensure$capitalized_name$IsMutable();\n" " ensure$capitalized_name$IsMutable();\n"
" $name$_.add(value);\n" " $name$_.add(value);\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -895,7 +880,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers(
" for (int value : values) {\n" " for (int value : values) {\n"
" $name$_.add(value);\n" " $name$_.add(value);\n"
" }\n" " }\n"
" $on_changed$\n" " onChanged();\n"
" return this;\n" " return this;\n"
"}\n"); "}\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
@ -935,7 +920,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMergingCode(
" ensure$capitalized_name$IsMutable();\n" " ensure$capitalized_name$IsMutable();\n"
" $name$_.addAll(other.$name$_);\n" " $name$_.addAll(other.$name$_);\n"
" }\n" " }\n"
" $on_changed$\n" " onChanged();\n"
"}\n"); "}\n");
} }
@ -967,8 +952,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateParsingCode(
printer->Print( printer->Print(
variables_, variables_,
"int rawValue = input.readEnum();\n" "int rawValue = input.readEnum();\n"
"@SuppressWarnings(\"deprecation\")\n" "$type$ value = $type$.forNumber(rawValue);\n"
"$type$ value = $type$.$for_number$(rawValue);\n"
"if (value == null) {\n" "if (value == null) {\n"
" unknownFields.mergeVarintField($number$, rawValue);\n" " unknownFields.mergeVarintField($number$, rawValue);\n"
"} else {\n" "} else {\n"

@ -127,6 +127,8 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex,
if (valueJavaType == JAVATYPE_ENUM) { if (valueJavaType == JAVATYPE_ENUM) {
// We store enums as Integers internally. // We store enums as Integers internally.
(*variables)["value_type"] = "int"; (*variables)["value_type"] = "int";
(*variables)["value_type_pass_through_nullness"] =
(*variables)["value_type"];
(*variables)["boxed_value_type"] = "java.lang.Integer"; (*variables)["boxed_value_type"] = "java.lang.Integer";
(*variables)["value_wire_type"] = WireType(value); (*variables)["value_wire_type"] = WireType(value);
(*variables)["value_default_value"] = (*variables)["value_default_value"] =
@ -262,9 +264,10 @@ void ImmutableMapFieldGenerator::GenerateInterfaceMembers(
WriteFieldDocComment(printer, descriptor_); WriteFieldDocComment(printer, descriptor_);
printer->Print(variables_, printer->Print(variables_,
"$deprecation$\n" "$deprecation$\n"
"$value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" "$value_type_pass_through_nullness$ "
"${$get$capitalized_name$ValueOrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue);\n"); " $value_type_pass_through_nullness$ defaultValue);\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
WriteFieldDocComment(printer, descriptor_); WriteFieldDocComment(printer, descriptor_);
printer->Print(variables_, printer->Print(variables_,
@ -609,9 +612,10 @@ void ImmutableMapFieldGenerator::GenerateMapGetters(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" "public $value_type_pass_through_nullness$ "
"${$get$capitalized_name$ValueOrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue) {\n" " $value_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n"
" internalGet$capitalized_name$().getMap();\n" " internalGet$capitalized_name$().getMap();\n"
@ -661,9 +665,10 @@ void ImmutableMapFieldGenerator::GenerateMapGetters(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" "public $value_type_pass_through_nullness$ "
"${$get$capitalized_name$OrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue) {\n" " $value_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$type_parameters$> map =\n" " java.util.Map<$type_parameters$> map =\n"
" internalGet$capitalized_name$().getMap();\n" " internalGet$capitalized_name$().getMap();\n"

@ -126,6 +126,8 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex,
if (GetJavaType(value) == JAVATYPE_ENUM) { if (GetJavaType(value) == JAVATYPE_ENUM) {
// We store enums as Integers internally. // We store enums as Integers internally.
(*variables)["value_type"] = "int"; (*variables)["value_type"] = "int";
(*variables)["value_type_pass_through_nullness"] =
(*variables)["value_type"];
(*variables)["boxed_value_type"] = "java.lang.Integer"; (*variables)["boxed_value_type"] = "java.lang.Integer";
(*variables)["value_wire_type"] = WireType(value); (*variables)["value_wire_type"] = WireType(value);
(*variables)["value_default_value"] = (*variables)["value_default_value"] =
@ -247,9 +249,10 @@ void ImmutableMapFieldLiteGenerator::GenerateInterfaceMembers(
WriteFieldDocComment(printer, descriptor_); WriteFieldDocComment(printer, descriptor_);
printer->Print(variables_, printer->Print(variables_,
"$deprecation$\n" "$deprecation$\n"
"$value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" "$value_type_pass_through_nullness$ "
"${$get$capitalized_name$ValueOrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue);\n"); " $value_type_pass_through_nullness$ defaultValue);\n");
printer->Annotate("{", "}", descriptor_); printer->Annotate("{", "}", descriptor_);
WriteFieldDocComment(printer, descriptor_); WriteFieldDocComment(printer, descriptor_);
printer->Print(variables_, printer->Print(variables_,
@ -373,9 +376,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_enum_type$ ${$get$capitalized_name$OrDefault$}$(\n" "public $value_enum_type_pass_through_nullness$ "
"${$get$capitalized_name$OrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_enum_type$ defaultValue) {\n" " $value_enum_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n"
" internalGet$capitalized_name$();\n" " internalGet$capitalized_name$();\n"
@ -429,9 +433,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" "public $value_type_pass_through_nullness$ "
"${$get$capitalized_name$ValueOrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue) {\n" " $value_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n"
" internalGet$capitalized_name$();\n" " internalGet$capitalized_name$();\n"
@ -482,9 +487,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" "public $value_type_pass_through_nullness$ "
"${$get$capitalized_name$OrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue) {\n" " $value_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$type_parameters$> map =\n" " java.util.Map<$type_parameters$> map =\n"
" internalGet$capitalized_name$();\n" " internalGet$capitalized_name$();\n"
@ -701,9 +707,10 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" "public $value_type_pass_through_nullness$ "
"${$get$capitalized_name$ValueOrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue) {\n" " $value_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n"
" instance.get$capitalized_name$ValueMap();\n" " instance.get$capitalized_name$ValueMap();\n"
@ -776,9 +783,10 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers(
variables_, variables_,
"@java.lang.Override\n" "@java.lang.Override\n"
"$deprecation$\n" "$deprecation$\n"
"public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" "public $value_type_pass_through_nullness$ "
"${$get$capitalized_name$OrDefault$}$(\n"
" $key_type$ key,\n" " $key_type$ key,\n"
" $value_type$ defaultValue) {\n" " $value_type_pass_through_nullness$ defaultValue) {\n"
" $key_null_check$\n" " $key_null_check$\n"
" java.util.Map<$type_parameters$> map =\n" " java.util.Map<$type_parameters$> map =\n"
" instance.get$capitalized_name$Map();\n" " instance.get$capitalized_name$Map();\n"

@ -425,6 +425,8 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
vars["oneof_capitalized_name"] = vars["oneof_capitalized_name"] =
context_->GetOneofGeneratorInfo(oneof)->capitalized_name; context_->GetOneofGeneratorInfo(oneof)->capitalized_name;
vars["oneof_index"] = StrCat((oneof)->index()); vars["oneof_index"] = StrCat((oneof)->index());
vars["{"] = "";
vars["}"] = "";
// oneofCase_ and oneof_ // oneofCase_ and oneof_
printer->Print(vars, printer->Print(vars,
"private int $oneof_name$Case_ = 0;\n" "private int $oneof_name$Case_ = 0;\n"
@ -432,11 +434,12 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
// OneofCase enum // OneofCase enum
printer->Print( printer->Print(
vars, vars,
"public enum $oneof_capitalized_name$Case\n" "public enum ${$$oneof_capitalized_name$Case$}$\n"
// TODO(dweis): Remove EnumLite when we want to break compatibility with // TODO(dweis): Remove EnumLite when we want to break compatibility with
// 3.x users // 3.x users
" implements com.google.protobuf.Internal.EnumLite,\n" " implements com.google.protobuf.Internal.EnumLite,\n"
" com.google.protobuf.AbstractMessage.InternalOneOfEnum {\n"); " com.google.protobuf.AbstractMessage.InternalOneOfEnum {\n");
printer->Annotate("{", "}", oneof);
printer->Indent(); printer->Indent();
for (int j = 0; j < (oneof)->field_count(); j++) { for (int j = 0; j < (oneof)->field_count(); j++) {
const FieldDescriptor* field = (oneof)->field(j); const FieldDescriptor* field = (oneof)->field(j);
@ -445,6 +448,7 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
field->options().deprecated() ? "@java.lang.Deprecated " : "", field->options().deprecated() ? "@java.lang.Deprecated " : "",
"field_name", ToUpper(field->name()), "field_number", "field_name", ToUpper(field->name()), "field_number",
StrCat(field->number())); StrCat(field->number()));
printer->Annotate("field_name", field);
} }
printer->Print("$cap_oneof_name$_NOT_SET(0);\n", "cap_oneof_name", printer->Print("$cap_oneof_name$_NOT_SET(0);\n", "cap_oneof_name",
ToUpper(vars["oneof_name"])); ToUpper(vars["oneof_name"]));
@ -487,11 +491,12 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) {
// oneofCase() // oneofCase()
printer->Print(vars, printer->Print(vars,
"public $oneof_capitalized_name$Case\n" "public $oneof_capitalized_name$Case\n"
"get$oneof_capitalized_name$Case() {\n" "${$get$oneof_capitalized_name$Case$}$() {\n"
" return $oneof_capitalized_name$Case.forNumber(\n" " return $oneof_capitalized_name$Case.forNumber(\n"
" $oneof_name$Case_);\n" " $oneof_name$Case_);\n"
"}\n" "}\n"
"\n"); "\n");
printer->Annotate("{", "}", oneof);
} }
if (IsAnyMessage(descriptor_)) { if (IsAnyMessage(descriptor_)) {
@ -1012,6 +1017,8 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode(
" return true;\n" " return true;\n"
"}\n" "}\n"
"if (!(obj instanceof $classname$)) {\n" "if (!(obj instanceof $classname$)) {\n"
// don't simply return false because mutable and immutable types
// can be equal
" return super.equals(obj);\n" " return super.equals(obj);\n"
"}\n" "}\n"
"$classname$ other = ($classname$) obj;\n" "$classname$ other = ($classname$) obj;\n"

@ -234,6 +234,8 @@ constexpr int CodeGeneratorResponse::Feature_ARRAYSIZE;
class Version::_Internal { class Version::_Internal {
public: public:
using HasBits = decltype(std::declval<Version>()._impl_._has_bits_); using HasBits = decltype(std::declval<Version>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(Version, _impl_._has_bits_);
static void set_has_major(HasBits* has_bits) { static void set_has_major(HasBits* has_bits) {
(*has_bits)[0] |= 2u; (*has_bits)[0] |= 2u;
} }
@ -558,6 +560,8 @@ void Version::InternalSwap(Version* other) {
class CodeGeneratorRequest::_Internal { class CodeGeneratorRequest::_Internal {
public: public:
using HasBits = decltype(std::declval<CodeGeneratorRequest>()._impl_._has_bits_); using HasBits = decltype(std::declval<CodeGeneratorRequest>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorRequest, _impl_._has_bits_);
static void set_has_parameter(HasBits* has_bits) { static void set_has_parameter(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -910,6 +914,8 @@ void CodeGeneratorRequest::InternalSwap(CodeGeneratorRequest* other) {
class CodeGeneratorResponse_File::_Internal { class CodeGeneratorResponse_File::_Internal {
public: public:
using HasBits = decltype(std::declval<CodeGeneratorResponse_File>()._impl_._has_bits_); using HasBits = decltype(std::declval<CodeGeneratorResponse_File>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorResponse_File, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -1300,6 +1306,8 @@ void CodeGeneratorResponse_File::InternalSwap(CodeGeneratorResponse_File* other)
class CodeGeneratorResponse::_Internal { class CodeGeneratorResponse::_Internal {
public: public:
using HasBits = decltype(std::declval<CodeGeneratorResponse>()._impl_._has_bits_); using HasBits = decltype(std::declval<CodeGeneratorResponse>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorResponse, _impl_._has_bits_);
static void set_has_error(HasBits* has_bits) { static void set_has_error(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }

@ -1512,6 +1512,8 @@ void FileDescriptorSet::InternalSwap(FileDescriptorSet* other) {
class FileDescriptorProto::_Internal { class FileDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<FileDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<FileDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(FileDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -2209,6 +2211,8 @@ void FileDescriptorProto::InternalSwap(FileDescriptorProto* other) {
class DescriptorProto_ExtensionRange::_Internal { class DescriptorProto_ExtensionRange::_Internal {
public: public:
using HasBits = decltype(std::declval<DescriptorProto_ExtensionRange>()._impl_._has_bits_); using HasBits = decltype(std::declval<DescriptorProto_ExtensionRange>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(DescriptorProto_ExtensionRange, _impl_._has_bits_);
static void set_has_start(HasBits* has_bits) { static void set_has_start(HasBits* has_bits) {
(*has_bits)[0] |= 2u; (*has_bits)[0] |= 2u;
} }
@ -2493,6 +2497,8 @@ void DescriptorProto_ExtensionRange::InternalSwap(DescriptorProto_ExtensionRange
class DescriptorProto_ReservedRange::_Internal { class DescriptorProto_ReservedRange::_Internal {
public: public:
using HasBits = decltype(std::declval<DescriptorProto_ReservedRange>()._impl_._has_bits_); using HasBits = decltype(std::declval<DescriptorProto_ReservedRange>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(DescriptorProto_ReservedRange, _impl_._has_bits_);
static void set_has_start(HasBits* has_bits) { static void set_has_start(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -2730,6 +2736,8 @@ void DescriptorProto_ReservedRange::InternalSwap(DescriptorProto_ReservedRange*
class DescriptorProto::_Internal { class DescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<DescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<DescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(DescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -3505,6 +3513,8 @@ void ExtensionRangeOptions::InternalSwap(ExtensionRangeOptions* other) {
class FieldDescriptorProto::_Internal { class FieldDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<FieldDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<FieldDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(FieldDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -4186,6 +4196,8 @@ void FieldDescriptorProto::InternalSwap(FieldDescriptorProto* other) {
class OneofDescriptorProto::_Internal { class OneofDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<OneofDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<OneofDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(OneofDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -4461,6 +4473,8 @@ void OneofDescriptorProto::InternalSwap(OneofDescriptorProto* other) {
class EnumDescriptorProto_EnumReservedRange::_Internal { class EnumDescriptorProto_EnumReservedRange::_Internal {
public: public:
using HasBits = decltype(std::declval<EnumDescriptorProto_EnumReservedRange>()._impl_._has_bits_); using HasBits = decltype(std::declval<EnumDescriptorProto_EnumReservedRange>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(EnumDescriptorProto_EnumReservedRange, _impl_._has_bits_);
static void set_has_start(HasBits* has_bits) { static void set_has_start(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -4698,6 +4712,8 @@ void EnumDescriptorProto_EnumReservedRange::InternalSwap(EnumDescriptorProto_Enu
class EnumDescriptorProto::_Internal { class EnumDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<EnumDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<EnumDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(EnumDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -5084,6 +5100,8 @@ void EnumDescriptorProto::InternalSwap(EnumDescriptorProto* other) {
class EnumValueDescriptorProto::_Internal { class EnumValueDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<EnumValueDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<EnumValueDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(EnumValueDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -5395,6 +5413,8 @@ void EnumValueDescriptorProto::InternalSwap(EnumValueDescriptorProto* other) {
class ServiceDescriptorProto::_Internal { class ServiceDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<ServiceDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<ServiceDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(ServiceDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -5706,6 +5726,8 @@ void ServiceDescriptorProto::InternalSwap(ServiceDescriptorProto* other) {
class MethodDescriptorProto::_Internal { class MethodDescriptorProto::_Internal {
public: public:
using HasBits = decltype(std::declval<MethodDescriptorProto>()._impl_._has_bits_); using HasBits = decltype(std::declval<MethodDescriptorProto>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(MethodDescriptorProto, _impl_._has_bits_);
static void set_has_name(HasBits* has_bits) { static void set_has_name(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -6163,6 +6185,8 @@ void MethodDescriptorProto::InternalSwap(MethodDescriptorProto* other) {
class FileOptions::_Internal { class FileOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<FileOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<FileOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(FileOptions, _impl_._has_bits_);
static void set_has_java_package(HasBits* has_bits) { static void set_has_java_package(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -7282,6 +7306,8 @@ void FileOptions::InternalSwap(FileOptions* other) {
class MessageOptions::_Internal { class MessageOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<MessageOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<MessageOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(MessageOptions, _impl_._has_bits_);
static void set_has_message_set_wire_format(HasBits* has_bits) { static void set_has_message_set_wire_format(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -7630,6 +7656,8 @@ void MessageOptions::InternalSwap(MessageOptions* other) {
class FieldOptions::_Internal { class FieldOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<FieldOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<FieldOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(FieldOptions, _impl_._has_bits_);
static void set_has_ctype(HasBits* has_bits) { static void set_has_ctype(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -8286,6 +8314,8 @@ void OneofOptions::InternalSwap(OneofOptions* other) {
class EnumOptions::_Internal { class EnumOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<EnumOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<EnumOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(EnumOptions, _impl_._has_bits_);
static void set_has_allow_alias(HasBits* has_bits) { static void set_has_allow_alias(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -8578,6 +8608,8 @@ void EnumOptions::InternalSwap(EnumOptions* other) {
class EnumValueOptions::_Internal { class EnumValueOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<EnumValueOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<EnumValueOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(EnumValueOptions, _impl_._has_bits_);
static void set_has_deprecated(HasBits* has_bits) { static void set_has_deprecated(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -8827,6 +8859,8 @@ void EnumValueOptions::InternalSwap(EnumValueOptions* other) {
class ServiceOptions::_Internal { class ServiceOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<ServiceOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<ServiceOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(ServiceOptions, _impl_._has_bits_);
static void set_has_deprecated(HasBits* has_bits) { static void set_has_deprecated(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -9076,6 +9110,8 @@ void ServiceOptions::InternalSwap(ServiceOptions* other) {
class MethodOptions::_Internal { class MethodOptions::_Internal {
public: public:
using HasBits = decltype(std::declval<MethodOptions>()._impl_._has_bits_); using HasBits = decltype(std::declval<MethodOptions>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(MethodOptions, _impl_._has_bits_);
static void set_has_deprecated(HasBits* has_bits) { static void set_has_deprecated(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -9377,6 +9413,8 @@ void MethodOptions::InternalSwap(MethodOptions* other) {
class UninterpretedOption_NamePart::_Internal { class UninterpretedOption_NamePart::_Internal {
public: public:
using HasBits = decltype(std::declval<UninterpretedOption_NamePart>()._impl_._has_bits_); using HasBits = decltype(std::declval<UninterpretedOption_NamePart>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(UninterpretedOption_NamePart, _impl_._has_bits_);
static void set_has_name_part(HasBits* has_bits) { static void set_has_name_part(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -9653,6 +9691,8 @@ void UninterpretedOption_NamePart::InternalSwap(UninterpretedOption_NamePart* ot
class UninterpretedOption::_Internal { class UninterpretedOption::_Internal {
public: public:
using HasBits = decltype(std::declval<UninterpretedOption>()._impl_._has_bits_); using HasBits = decltype(std::declval<UninterpretedOption>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(UninterpretedOption, _impl_._has_bits_);
static void set_has_identifier_value(HasBits* has_bits) { static void set_has_identifier_value(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -10122,6 +10162,8 @@ void UninterpretedOption::InternalSwap(UninterpretedOption* other) {
class SourceCodeInfo_Location::_Internal { class SourceCodeInfo_Location::_Internal {
public: public:
using HasBits = decltype(std::declval<SourceCodeInfo_Location>()._impl_._has_bits_); using HasBits = decltype(std::declval<SourceCodeInfo_Location>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(SourceCodeInfo_Location, _impl_._has_bits_);
static void set_has_leading_comments(HasBits* has_bits) { static void set_has_leading_comments(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }
@ -10716,6 +10758,8 @@ void SourceCodeInfo::InternalSwap(SourceCodeInfo* other) {
class GeneratedCodeInfo_Annotation::_Internal { class GeneratedCodeInfo_Annotation::_Internal {
public: public:
using HasBits = decltype(std::declval<GeneratedCodeInfo_Annotation>()._impl_._has_bits_); using HasBits = decltype(std::declval<GeneratedCodeInfo_Annotation>()._impl_._has_bits_);
static constexpr int32_t kHasBitsOffset =
8 * PROTOBUF_FIELD_OFFSET(GeneratedCodeInfo_Annotation, _impl_._has_bits_);
static void set_has_source_file(HasBits* has_bits) { static void set_has_source_file(HasBits* has_bits) {
(*has_bits)[0] |= 1u; (*has_bits)[0] |= 1u;
} }

@ -604,11 +604,8 @@ message FieldOptions {
// check its required fields, regardless of whether or not the message has // check its required fields, regardless of whether or not the message has
// been parsed. // been parsed.
// //
// As of 2021, lazy does no correctness checks on the byte stream during // As of May 2022, lazy verifies the contents of the byte stream during
// parsing. This may lead to crashes if and when an invalid byte stream is // parsing. An invalid byte stream will cause the overall parsing to fail.
// finally parsed upon access.
//
// TODO(b/211906113): Enable validation on lazy fields.
optional bool lazy = 5 [default = false]; optional bool lazy = 5 [default = false];
// unverified_lazy does no correctness checks on the byte stream. This should // unverified_lazy does no correctness checks on the byte stream. This should

@ -983,8 +983,6 @@ void Reflection::Swap(Message* message1, Message* message2) const {
return; return;
} }
GOOGLE_DCHECK_EQ(message1->GetOwningArena(), message2->GetOwningArena());
UnsafeArenaSwap(message1, message2); UnsafeArenaSwap(message1, message2);
} }
@ -1014,9 +1012,6 @@ void Reflection::SwapFieldsImpl(
std::set<int> swapped_oneof; std::set<int> swapped_oneof;
GOOGLE_DCHECK(!unsafe_shallow_swap || message1->GetArenaForAllocation() ==
message2->GetArenaForAllocation());
const Message* prototype = const Message* prototype =
message_factory_->GetPrototype(message1->GetDescriptor()); message_factory_->GetPrototype(message1->GetDescriptor());
for (const auto* field : fields) { for (const auto* field : fields) {
@ -1073,6 +1068,9 @@ void Reflection::SwapFields(
void Reflection::UnsafeShallowSwapFields( void Reflection::UnsafeShallowSwapFields(
Message* message1, Message* message2, Message* message1, Message* message2,
const std::vector<const FieldDescriptor*>& fields) const { const std::vector<const FieldDescriptor*>& fields) const {
GOOGLE_DCHECK_EQ(message1->GetArenaForAllocation(),
message2->GetArenaForAllocation());
SwapFieldsImpl<true>(message1, message2, fields); SwapFieldsImpl<true>(message1, message2, fields);
} }
@ -1103,6 +1101,11 @@ bool Reflection::HasField(const Message& message,
} }
void Reflection::UnsafeArenaSwap(Message* lhs, Message* rhs) const { void Reflection::UnsafeArenaSwap(Message* lhs, Message* rhs) const {
GOOGLE_DCHECK_EQ(lhs->GetOwningArena(), rhs->GetOwningArena());
InternalSwap(lhs, rhs);
}
void Reflection::InternalSwap(Message* lhs, Message* rhs) const {
if (lhs == rhs) return; if (lhs == rhs) return;
MutableInternalMetadata(lhs)->InternalSwap(MutableInternalMetadata(rhs)); MutableInternalMetadata(lhs)->InternalSwap(MutableInternalMetadata(rhs));

@ -120,6 +120,8 @@ struct Offset {
#pragma warning(disable : 4324) #pragma warning(disable : 4324)
#endif #endif
struct FieldAuxDefaultMessage {};
// Base class for message-level table with info for the tail-call parser. // Base class for message-level table with info for the tail-call parser.
struct alignas(uint64_t) TcParseTableBase { struct alignas(uint64_t) TcParseTableBase {
// Common attributes for message layout: // Common attributes for message layout:
@ -191,7 +193,7 @@ struct alignas(uint64_t) TcParseTableBase {
// Field entry for all fields. // Field entry for all fields.
struct FieldEntry { struct FieldEntry {
uint32_t offset; // offset in the message object uint32_t offset; // offset in the message object
int32_t has_idx; // has-bit index int32_t has_idx; // has-bit index, relative to the message object
uint16_t aux_idx; // index for `field_aux`. uint16_t aux_idx; // index for `field_aux`.
uint16_t type_card; // `FieldType` and `Cardinality` (see _impl.h) uint16_t type_card; // `FieldType` and `Cardinality` (see _impl.h)
}; };
@ -204,20 +206,28 @@ struct alignas(uint64_t) TcParseTableBase {
// Auxiliary entries for field types that need extra information. // Auxiliary entries for field types that need extra information.
union FieldAux { union FieldAux {
constexpr FieldAux() : message_default(nullptr) {} constexpr FieldAux() : message_default_p(nullptr) {}
constexpr FieldAux(bool (*enum_validator)(int)) constexpr FieldAux(bool (*enum_validator)(int))
: enum_validator(enum_validator) {} : enum_validator(enum_validator) {}
constexpr FieldAux(field_layout::Offset off) : offset(off.off) {} constexpr FieldAux(field_layout::Offset off) : offset(off.off) {}
constexpr FieldAux(int16_t range_start, uint16_t range_length) constexpr FieldAux(int16_t range_start, uint16_t range_length)
: enum_range{range_start, range_length} {} : enum_range{range_start, range_length} {}
constexpr FieldAux(const MessageLite* msg) : message_default(msg) {} constexpr FieldAux(const MessageLite* msg) : message_default_p(msg) {}
constexpr FieldAux(FieldAuxDefaultMessage, const void* msg)
: message_default_p(msg) {}
constexpr FieldAux(const TcParseTableBase* table) : table(table) {}
bool (*enum_validator)(int); bool (*enum_validator)(int);
struct { struct {
int16_t start; // minimum enum number (if it fits) int16_t start; // minimum enum number (if it fits)
uint16_t length; // length of range (i.e., max = start + length - 1) uint16_t length; // length of range (i.e., max = start + length - 1)
} enum_range; } enum_range;
uint32_t offset; uint32_t offset;
const MessageLite* message_default; const void* message_default_p;
const TcParseTableBase* table;
const MessageLite* message_default() const {
return static_cast<const MessageLite*>(message_default_p);
}
}; };
const FieldAux* field_aux(uint32_t idx) const { const FieldAux* field_aux(uint32_t idx) const {
return reinterpret_cast<const FieldAux*>(reinterpret_cast<uintptr_t>(this) + return reinterpret_cast<const FieldAux*>(reinterpret_cast<uintptr_t>(this) +

@ -147,6 +147,10 @@ enum TransformValidation : uint16_t {
// String fields: // String fields:
kTvUtf8Debug = 1 << kTvShift, // proto2 kTvUtf8Debug = 1 << kTvShift, // proto2
kTvUtf8 = 2 << kTvShift, // proto3 kTvUtf8 = 2 << kTvShift, // proto3
// Message fields:
kTvDefault = 1 << kTvShift, // Aux has default_instance
kTvTable = 2 << kTvShift, // Aux has TcParseTableBase*
}; };
static_assert((kTvEnum & kTvRange) != 0, static_assert((kTvEnum & kTvRange) != 0,
@ -262,6 +266,11 @@ extern template void AlignFail<8>(uintptr_t);
// TcParser implements most of the parsing logic for tailcall tables. // TcParser implements most of the parsing logic for tailcall tables.
class PROTOBUF_EXPORT TcParser final { class PROTOBUF_EXPORT TcParser final {
public: public:
template <typename T>
static constexpr const TcParseTableBase* GetTable() {
return &T::_table_.header;
}
static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL); static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL);
static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL); static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL);
@ -364,16 +373,26 @@ class PROTOBUF_EXPORT TcParser final {
// Functions referenced by generated fast tables (message types): // Functions referenced by generated fast tables (message types):
// M: message G: group // M: message G: group
// d: default* t: TcParseTable* (the contents of aux)
// S: singular R: repeated // S: singular R: repeated
// 1/2: tag length (bytes) // 1/2: tag length (bytes)
static const char* FastMS1(PROTOBUF_TC_PARAM_DECL); static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL);
static const char* FastMS2(PROTOBUF_TC_PARAM_DECL); static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL);
static const char* FastMR1(PROTOBUF_TC_PARAM_DECL); static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL);
static const char* FastMR2(PROTOBUF_TC_PARAM_DECL); static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL);
static const char* FastGS1(PROTOBUF_TC_PARAM_DECL); static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL);
static const char* FastGS2(PROTOBUF_TC_PARAM_DECL); static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL);
static const char* FastGR1(PROTOBUF_TC_PARAM_DECL); static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL);
static const char* FastGR2(PROTOBUF_TC_PARAM_DECL); static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL);
static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL);
static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL);
static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL);
static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL);
static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL);
static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL);
static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL);
static const char* FastGtR2(PROTOBUF_TC_PARAM_DECL);
template <typename T> template <typename T>
static inline T& RefAt(void* x, size_t offset) { static inline T& RefAt(void* x, size_t offset) {
@ -421,9 +440,9 @@ class PROTOBUF_EXPORT TcParser final {
private: private:
friend class GeneratedTcTableLiteTest; friend class GeneratedTcTableLiteTest;
template <typename TagType, bool group_coding> template <typename TagType, bool group_coding, bool aux_is_table>
static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType, bool group_coding> template <typename TagType, bool group_coding, bool aux_is_table>
static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits( static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits(
@ -432,7 +451,7 @@ class PROTOBUF_EXPORT TcParser final {
if (has_bits_offset) { if (has_bits_offset) {
// Only the first 32 has-bits are updated. Nothing above those is stored, // Only the first 32 has-bits are updated. Nothing above those is stored,
// but e.g. messages without has-bits update the upper bits. // but e.g. messages without has-bits update the upper bits.
RefAt<uint32_t>(msg, has_bits_offset) = static_cast<uint32_t>(hasbits); RefAt<uint32_t>(msg, has_bits_offset) |= static_cast<uint32_t>(hasbits);
} }
} }

@ -85,10 +85,7 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop(
const TcParseTableBase* table) { const TcParseTableBase* table) {
ScopedArenaSwap saved(msg, ctx); ScopedArenaSwap saved(msg, ctx);
while (!ctx->Done(&ptr)) { while (!ctx->Done(&ptr)) {
// Unconditionally read has bits, even if we don't have has bits. ptr = TagDispatch(msg, ptr, ctx, table, 0, {});
// has_bits_offset will be 0 and we will just read something valid.
uint64_t hasbits = ReadAt<uint32_t>(msg, table->has_bits_offset);
ptr = TagDispatch(msg, ptr, ctx, table, hasbits, {});
if (ptr == nullptr) break; if (ptr == nullptr) break;
if (ctx->LastTag() != 1) break; // Ended on terminating tag if (ctx->LastTag() != 1) break; // Ended on terminating tag
} }
@ -362,9 +359,9 @@ inline PROTOBUF_ALWAYS_INLINE void InvertPacked(TcFieldData& data) {
// Message fields // Message fields
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
template <typename TagType, bool group_coding> template <typename TagType, bool group_coding, bool aux_is_table>
inline PROTOBUF_ALWAYS_INLINE inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl(
const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS);
} }
@ -373,74 +370,139 @@ const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) {
hasbits |= (uint64_t{1} << data.hasbit_idx()); hasbits |= (uint64_t{1} << data.hasbit_idx());
SyncHasbits(msg, hasbits, table); SyncHasbits(msg, hasbits, table);
auto& field = RefAt<MessageLite*>(msg, data.offset()); auto& field = RefAt<MessageLite*>(msg, data.offset());
if (field == nullptr) {
const MessageLite* default_instance = if (aux_is_table) {
table->field_aux(data.aux_idx())->message_default; const auto* inner_table = table->field_aux(data.aux_idx())->table;
field = default_instance->New(ctx->data().arena); if (field == nullptr) {
} field = inner_table->default_instance->New(ctx->data().arena);
if (group_coding) { }
return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); if (group_coding) {
return ctx->ParseGroup<TcParser>(field, ptr, FastDecodeTag(saved_tag),
inner_table);
}
return ctx->ParseMessage<TcParser>(field, ptr, inner_table);
} else {
if (field == nullptr) {
const MessageLite* default_instance =
table->field_aux(data.aux_idx())->message_default();
field = default_instance->New(ctx->data().arena);
}
if (group_coding) {
return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag));
}
return ctx->ParseMessage(field, ptr);
} }
return ctx->ParseMessage(field, ptr);
} }
const char* TcParser::FastMS1(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastMdS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, false>( PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastMdS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastGdS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastGdS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastMtS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, false, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
const char* TcParser::FastMS2(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastMtS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, false>( PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, false, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
const char* TcParser::FastGS1(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastGtS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, true>( PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, true, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
const char* TcParser::FastGS2(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, true>( PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, true, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
template <typename TagType, bool group_coding> template <typename TagType, bool group_coding, bool aux_is_table>
inline PROTOBUF_ALWAYS_INLINE inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
const char* TcParser::RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS);
} }
auto saved_tag = UnalignedLoad<TagType>(ptr); auto saved_tag = UnalignedLoad<TagType>(ptr);
ptr += sizeof(TagType); ptr += sizeof(TagType);
SyncHasbits(msg, hasbits, table); SyncHasbits(msg, hasbits, table);
const MessageLite* default_instance = auto* aux = table->field_aux(data.aux_idx());
table->field_aux(data.aux_idx())->message_default; if (aux_is_table) {
auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset()); auto* inner_table = aux->table;
MessageLite* submsg = auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset());
field.Add<GenericTypeHandler<MessageLite>>(default_instance); MessageLite* submsg = field.Add<GenericTypeHandler<MessageLite>>(
if (group_coding) { inner_table->default_instance);
return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); if (group_coding) {
return ctx->ParseGroup<TcParser>(submsg, ptr, FastDecodeTag(saved_tag),
inner_table);
}
return ctx->ParseMessage<TcParser>(submsg, ptr, inner_table);
} else {
const MessageLite* default_instance = aux->message_default();
auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset());
MessageLite* submsg =
field.Add<GenericTypeHandler<MessageLite>>(default_instance);
if (group_coding) {
return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag));
}
return ctx->ParseMessage(submsg, ptr);
} }
return ctx->ParseMessage(submsg, ptr);
} }
const char* TcParser::FastMR1(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, false>( PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastMdR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastGdR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastGdR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, true, false>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
const char* TcParser::FastMR2(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastMtR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, false>( PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, false, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
const char* TcParser::FastGR1(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastMtR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, true>( PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, false, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
const char* TcParser::FastGR2(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::FastGtR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, true>( PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, true, true>(
PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastGtR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, true, true>(
PROTOBUF_TC_PARAM_PASS); PROTOBUF_TC_PARAM_PASS);
} }
@ -977,10 +1039,6 @@ const char* TcParser::FastZ64P2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_NOINLINE const char* TcParser::FastUnknownEnumFallback( PROTOBUF_NOINLINE const char* TcParser::FastUnknownEnumFallback(
PROTOBUF_TC_PARAM_DECL) { PROTOBUF_TC_PARAM_DECL) {
(void)msg;
(void)ctx;
(void)hasbits;
// If we know we want to put this field directly into the unknown field set, // If we know we want to put this field directly into the unknown field set,
// then we can skip the call to MiniParse and directly call table->fallback. // then we can skip the call to MiniParse and directly call table->fallback.
// However, we first have to update `data` to contain the decoded tag. // However, we first have to update `data` to contain the decoded tag.
@ -1261,20 +1319,14 @@ const char* TcParser::FastUR2(PROTOBUF_TC_PARAM_DECL) {
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
namespace { namespace {
inline void SetHas(const TcParseTableBase* table, const FieldEntry& entry, inline void SetHas(const FieldEntry& entry, MessageLite* msg) {
MessageLite* msg, uint64_t& hasbits) { auto has_idx = static_cast<uint32_t>(entry.has_idx);
int32_t has_idx = entry.has_idx;
if (has_idx < 32) {
hasbits |= uint64_t{1} << has_idx;
} else {
auto* hasblocks = &TcParser::RefAt<uint32_t>(msg, table->has_bits_offset);
#if defined(__x86_64__) && defined(__GNUC__) #if defined(__x86_64__) && defined(__GNUC__)
asm("bts %1, %0\n" : "+m"(*hasblocks) : "r"(has_idx)); asm("bts %1, %0\n" : "+m"(*msg) : "r"(has_idx));
#else #else
auto& hasblock = hasblocks[has_idx / 32]; auto& hasblock = TcParser::RefAt<uint32_t>(msg, has_idx / 32 * 4);
hasblock |= uint32_t{1} << (has_idx % 32); hasblock |= uint32_t{1} << (has_idx % 32);
#endif #endif
}
} }
} // namespace } // namespace
@ -1364,7 +1416,7 @@ const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) {
} }
// Set the field present: // Set the field present:
if (card == field_layout::kFcOptional) { if (card == field_layout::kFcOptional) {
SetHas(table, entry, msg, hasbits); SetHas(entry, msg);
} else if (card == field_layout::kFcOneof) { } else if (card == field_layout::kFcOneof) {
ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); ChangeOneof(table, entry, data.tag() >> 3, ctx, msg);
} }
@ -1501,7 +1553,7 @@ const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) {
// Mark the field as present: // Mark the field as present:
const bool is_oneof = card == field_layout::kFcOneof; const bool is_oneof = card == field_layout::kFcOneof;
if (card == field_layout::kFcOptional) { if (card == field_layout::kFcOptional) {
SetHas(table, entry, msg, hasbits); SetHas(entry, msg);
} else if (is_oneof) { } else if (is_oneof) {
ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); ChangeOneof(table, entry, data.tag() >> 3, ctx, msg);
} }
@ -1676,7 +1728,7 @@ const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) {
const bool is_oneof = card == field_layout::kFcOneof; const bool is_oneof = card == field_layout::kFcOneof;
bool need_init = false; bool need_init = false;
if (card == field_layout::kFcOptional) { if (card == field_layout::kFcOptional) {
SetHas(table, entry, msg, hasbits); SetHas(entry, msg);
} else if (is_oneof) { } else if (is_oneof) {
need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg);
} }
@ -1789,21 +1841,31 @@ const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
const bool is_oneof = card == field_layout::kFcOneof; const bool is_oneof = card == field_layout::kFcOneof;
bool need_init = false; bool need_init = false;
if (card == field_layout::kFcOptional) { if (card == field_layout::kFcOptional) {
SetHas(table, entry, msg, hasbits); SetHas(entry, msg);
} else if (is_oneof) { } else if (is_oneof) {
need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg);
} }
MessageLite*& field = RefAt<MessageLite*>(msg, entry.offset);
if (need_init || field == nullptr) {
const MessageLite* default_instance =
table->field_aux(&entry)->message_default;
field = default_instance->New(ctx->data().arena);
}
SyncHasbits(msg, hasbits, table); SyncHasbits(msg, hasbits, table);
if (is_group) { MessageLite*& field = RefAt<MessageLite*>(msg, entry.offset);
return ctx->ParseGroup(field, ptr, decoded_tag); if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) {
auto* inner_table = table->field_aux(&entry)->table;
if (need_init || field == nullptr) {
field = inner_table->default_instance->New(ctx->data().arena);
}
if (is_group) {
return ctx->ParseGroup<TcParser>(field, ptr, decoded_tag, inner_table);
}
return ctx->ParseMessage<TcParser>(field, ptr, inner_table);
} else {
if (need_init || field == nullptr) {
field =
table->field_aux(&entry)->message_default()->New(ctx->data().arena);
}
if (is_group) {
return ctx->ParseGroup(field, ptr, decoded_tag);
}
return ctx->ParseMessage(field, ptr);
} }
return ctx->ParseMessage(field, ptr);
} }
const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) {
@ -1837,15 +1899,24 @@ const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) {
} }
SyncHasbits(msg, hasbits, table); SyncHasbits(msg, hasbits, table);
const MessageLite* default_instance = if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) {
table->field_aux(&entry)->message_default; auto* inner_table = table->field_aux(&entry)->table;
auto& field = RefAt<RepeatedPtrFieldBase>(msg, entry.offset); auto& field = RefAt<RepeatedPtrFieldBase>(msg, entry.offset);
MessageLite* value = MessageLite* value = field.Add<GenericTypeHandler<MessageLite>>(
field.Add<GenericTypeHandler<MessageLite>>(default_instance); inner_table->default_instance);
if (is_group) { if (is_group) {
return ctx->ParseGroup(value, ptr, decoded_tag); return ctx->ParseGroup<TcParser>(value, ptr, decoded_tag, inner_table);
} }
return ctx->ParseMessage(value, ptr); return ctx->ParseMessage<TcParser>(value, ptr, inner_table);
} else {
auto& field = RefAt<RepeatedPtrFieldBase>(msg, entry.offset);
MessageLite* value = field.Add<GenericTypeHandler<MessageLite>>(
table->field_aux(&entry)->message_default());
if (is_group) {
return ctx->ParseGroup(value, ptr, decoded_tag);
}
return ctx->ParseMessage(value, ptr);
}
} }
const char* TcParser::MpMap(PROTOBUF_TC_PARAM_DECL) { const char* TcParser::MpMap(PROTOBUF_TC_PARAM_DECL) {

@ -109,7 +109,6 @@
#ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
#define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
#include <assert.h> #include <assert.h>
#include <atomic> #include <atomic>

@ -43,7 +43,6 @@
#ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__ #ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
#define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__ #define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/zero_copy_stream.h> #include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/port.h> #include <google/protobuf/port.h>

@ -37,7 +37,6 @@
#ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__ #ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__
#define GOOGLE_PROTOBUF_IO_PRINTER_H__ #define GOOGLE_PROTOBUF_IO_PRINTER_H__
#include <map> #include <map>
#include <string> #include <string>
#include <vector> #include <vector>

@ -37,7 +37,6 @@
#ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__ #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__
#define GOOGLE_PROTOBUF_IO_TOKENIZER_H__ #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__
#include <string> #include <string>
#include <vector> #include <vector>

@ -107,7 +107,6 @@
#ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__
#define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>

@ -40,7 +40,6 @@
#ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__
#define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__
#include <iosfwd> #include <iosfwd>
#include <string> #include <string>

@ -36,12 +36,16 @@
#include <algorithm> #include <algorithm>
#include <limits> #include <limits>
#include <utility>
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/casts.h> #include <google/protobuf/stubs/casts.h>
#include <google/protobuf/stubs/stl_util.h> #include <google/protobuf/stubs/stl_util.h>
// Must be included last
#include <google/protobuf/port_def.inc>
namespace google { namespace google {
namespace protobuf { namespace protobuf {
namespace io { namespace io {

@ -44,10 +44,10 @@
#ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
#define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__
#include <iosfwd> #include <iosfwd>
#include <memory> #include <memory>
#include <string> #include <string>
#include <utility>
#include <google/protobuf/stubs/callback.h> #include <google/protobuf/stubs/callback.h>
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>
@ -78,6 +78,10 @@ class PROTOBUF_EXPORT ArrayInputStream PROTOBUF_FUTURE_FINAL
ArrayInputStream(const void* data, int size, int block_size = -1); ArrayInputStream(const void* data, int size, int block_size = -1);
~ArrayInputStream() override = default; ~ArrayInputStream() override = default;
// `ArrayInputStream` is neither copiable nor assignable
ArrayInputStream(const ArrayInputStream&) = delete;
ArrayInputStream& operator=(const ArrayInputStream&) = delete;
// implements ZeroCopyInputStream ---------------------------------- // implements ZeroCopyInputStream ----------------------------------
bool Next(const void** data, int* size) override; bool Next(const void** data, int* size) override;
void BackUp(int count) override; void BackUp(int count) override;
@ -93,8 +97,6 @@ class PROTOBUF_EXPORT ArrayInputStream PROTOBUF_FUTURE_FINAL
int position_; int position_;
int last_returned_size_; // How many bytes we returned last time Next() int last_returned_size_; // How many bytes we returned last time Next()
// was called (used for error checking only). // was called (used for error checking only).
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayInputStream);
}; };
// =================================================================== // ===================================================================
@ -113,6 +115,10 @@ class PROTOBUF_EXPORT ArrayOutputStream PROTOBUF_FUTURE_FINAL
ArrayOutputStream(void* data, int size, int block_size = -1); ArrayOutputStream(void* data, int size, int block_size = -1);
~ArrayOutputStream() override = default; ~ArrayOutputStream() override = default;
// `ArrayOutputStream` is neither copiable nor assignable
ArrayOutputStream(const ArrayOutputStream&) = delete;
ArrayOutputStream& operator=(const ArrayOutputStream&) = delete;
// implements ZeroCopyOutputStream --------------------------------- // implements ZeroCopyOutputStream ---------------------------------
bool Next(void** data, int* size) override; bool Next(void** data, int* size) override;
void BackUp(int count) override; void BackUp(int count) override;
@ -126,8 +132,6 @@ class PROTOBUF_EXPORT ArrayOutputStream PROTOBUF_FUTURE_FINAL
int position_; int position_;
int last_returned_size_; // How many bytes we returned last time Next() int last_returned_size_; // How many bytes we returned last time Next()
// was called (used for error checking only). // was called (used for error checking only).
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayOutputStream);
}; };
// =================================================================== // ===================================================================
@ -148,6 +152,10 @@ class PROTOBUF_EXPORT StringOutputStream PROTOBUF_FUTURE_FINAL
explicit StringOutputStream(std::string* target); explicit StringOutputStream(std::string* target);
~StringOutputStream() override = default; ~StringOutputStream() override = default;
// `StringOutputStream` is neither copiable nor assignable
StringOutputStream(const StringOutputStream&) = delete;
StringOutputStream& operator=(const StringOutputStream&) = delete;
// implements ZeroCopyOutputStream --------------------------------- // implements ZeroCopyOutputStream ---------------------------------
bool Next(void** data, int* size) override; bool Next(void** data, int* size) override;
void BackUp(int count) override; void BackUp(int count) override;
@ -157,8 +165,6 @@ class PROTOBUF_EXPORT StringOutputStream PROTOBUF_FUTURE_FINAL
static constexpr size_t kMinimumSize = 16; static constexpr size_t kMinimumSize = 16;
std::string* target_; std::string* target_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream);
}; };
// Note: There is no StringInputStream. Instead, just create an // Note: There is no StringInputStream. Instead, just create an
@ -215,6 +221,10 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream {
int block_size = -1); int block_size = -1);
~CopyingInputStreamAdaptor() override; ~CopyingInputStreamAdaptor() override;
// `CopyingInputStreamAdaptor` is neither copiable nor assignable
CopyingInputStreamAdaptor(const CopyingInputStreamAdaptor&) = delete;
CopyingInputStreamAdaptor& operator=(const CopyingInputStreamAdaptor&) = delete;
// Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to // Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to
// delete the underlying CopyingInputStream when it is destroyed. // delete the underlying CopyingInputStream when it is destroyed.
void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; } void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; }
@ -255,8 +265,6 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream {
// BackUp(). These need to be returned again. // BackUp(). These need to be returned again.
// 0 <= backup_bytes_ <= buffer_used_ // 0 <= backup_bytes_ <= buffer_used_
int backup_bytes_; int backup_bytes_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingInputStreamAdaptor);
}; };
// =================================================================== // ===================================================================
@ -298,6 +306,10 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream {
int block_size = -1); int block_size = -1);
~CopyingOutputStreamAdaptor() override; ~CopyingOutputStreamAdaptor() override;
// `CopyingOutputStreamAdaptor` is neither copiable nor assignable
CopyingOutputStreamAdaptor(const CopyingOutputStreamAdaptor&) = delete;
CopyingOutputStreamAdaptor& operator=(const CopyingOutputStreamAdaptor&) = delete;
// Writes all pending data to the underlying stream. Returns false if a // Writes all pending data to the underlying stream. Returns false if a
// write error occurred on the underlying stream. (The underlying // write error occurred on the underlying stream. (The underlying
// stream itself is not necessarily flushed.) // stream itself is not necessarily flushed.)
@ -342,8 +354,6 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream {
// returned by Next()). When BackUp() is called, we just reduce this. // returned by Next()). When BackUp() is called, we just reduce this.
// 0 <= buffer_used_ <= buffer_size_. // 0 <= buffer_used_ <= buffer_size_.
int buffer_used_; int buffer_used_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOutputStreamAdaptor);
}; };
// =================================================================== // ===================================================================
@ -356,6 +366,10 @@ class PROTOBUF_EXPORT LimitingInputStream PROTOBUF_FUTURE_FINAL
LimitingInputStream(ZeroCopyInputStream* input, int64_t limit); LimitingInputStream(ZeroCopyInputStream* input, int64_t limit);
~LimitingInputStream() override; ~LimitingInputStream() override;
// `LimitingInputStream` is neither copiable nor assignable
LimitingInputStream(const LimitingInputStream&) = delete;
LimitingInputStream& operator=(const LimitingInputStream&) = delete;
// implements ZeroCopyInputStream ---------------------------------- // implements ZeroCopyInputStream ----------------------------------
bool Next(const void** data, int* size) override; bool Next(const void** data, int* size) override;
void BackUp(int count) override; void BackUp(int count) override;
@ -367,8 +381,6 @@ class PROTOBUF_EXPORT LimitingInputStream PROTOBUF_FUTURE_FINAL
ZeroCopyInputStream* input_; ZeroCopyInputStream* input_;
int64_t limit_; // Decreases as we go, becomes negative if we overshoot. int64_t limit_; // Decreases as we go, becomes negative if we overshoot.
int64_t prior_bytes_read_; // Bytes read on underlying stream at construction int64_t prior_bytes_read_; // Bytes read on underlying stream at construction
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream);
}; };

@ -61,6 +61,7 @@
#include <memory> #include <memory>
#include <sstream> #include <sstream>
#include <utility>
#include <google/protobuf/testing/file.h> #include <google/protobuf/testing/file.h>
#include <google/protobuf/test_util2.h> #include <google/protobuf/test_util2.h>
@ -570,7 +571,7 @@ TEST_F(IoTest, CompressionOptions) {
// Some ad-hoc testing of compression options. // Some ad-hoc testing of compression options.
std::string golden_filename = std::string golden_filename =
TestUtil::GetTestDataPath("net/proto2/internal/testdata/golden_message"); TestUtil::GetTestDataPath("third_party/protobuf/testdata/golden_message");
std::string golden; std::string golden;
GOOGLE_CHECK_OK(File::GetContents(golden_filename, &golden, true)); GOOGLE_CHECK_OK(File::GetContents(golden_filename, &golden, true));

@ -3749,7 +3749,7 @@ TEST(MapSerializationTest, DeterministicSubmessage) {
const std::string filename = "golden_message_maps"; const std::string filename = "golden_message_maps";
std::string golden; std::string golden;
GOOGLE_CHECK_OK(File::GetContents( GOOGLE_CHECK_OK(File::GetContents(
TestUtil::GetTestDataPath("net/proto2/internal/testdata/" + filename), TestUtil::GetTestDataPath("third_party/protobuf/testdata/" + filename),
&golden, true)); &golden, true));
t.ParseFromString(golden); t.ParseFromString(golden);
*(p.mutable_m()) = t; *(p.mutable_m()) = t;
@ -3791,7 +3791,7 @@ TEST(TextFormatMapTest, DynamicMessage) {
std::string expected_text; std::string expected_text;
GOOGLE_CHECK_OK( GOOGLE_CHECK_OK(
File::GetContents(TestUtil::GetTestDataPath("net/proto2/internal/" File::GetContents(TestUtil::GetTestDataPath("third_party/protobuf/"
"testdata/map_test_data.txt"), "testdata/map_test_data.txt"),
&expected_text, true)); &expected_text, true));
@ -3808,7 +3808,7 @@ TEST(TextFormatMapTest, Sorted) {
std::string expected_text; std::string expected_text;
GOOGLE_CHECK_OK( GOOGLE_CHECK_OK(
File::GetContents(TestUtil::GetTestDataPath("net/proto2/internal/" File::GetContents(TestUtil::GetTestDataPath("third_party/protobuf/"
"testdata/map_test_data.txt"), "testdata/map_test_data.txt"),
&expected_text, true)); &expected_text, true));
@ -3828,10 +3828,10 @@ TEST(TextFormatMapTest, Sorted) {
TEST(TextFormatMapTest, ParseCorruptedString) { TEST(TextFormatMapTest, ParseCorruptedString) {
std::string serialized_message; std::string serialized_message;
GOOGLE_CHECK_OK( GOOGLE_CHECK_OK(File::GetContents(
File::GetContents(TestUtil::GetTestDataPath( TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_maps"), "third_party/protobuf/testdata/golden_message_maps"),
&serialized_message, true)); &serialized_message, true));
UNITTEST::TestMaps message; UNITTEST::TestMaps message;
GOOGLE_CHECK(message.ParseFromString(serialized_message)); GOOGLE_CHECK(message.ParseFromString(serialized_message));
TestParseCorruptedString<UNITTEST::TestMaps, true>(message); TestParseCorruptedString<UNITTEST::TestMaps, true>(message);

@ -1197,6 +1197,8 @@ class PROTOBUF_EXPORT Reflection final {
void SwapOneofField(Message* lhs, Message* rhs, void SwapOneofField(Message* lhs, Message* rhs,
const OneofDescriptor* oneof_descriptor) const; const OneofDescriptor* oneof_descriptor) const;
void InternalSwap(Message* lhs, Message* rhs) const;
inline bool HasOneofField(const Message& message, inline bool HasOneofField(const Message& message,
const FieldDescriptor* field) const; const FieldDescriptor* field) const;
inline void SetOneofCase(Message* message, inline void SetOneofCase(Message* message,

@ -38,6 +38,7 @@
#include <climits> #include <climits>
#include <cstdint> #include <cstdint>
#include <string> #include <string>
#include <utility>
#include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/common.h>

@ -124,7 +124,7 @@ TEST(MESSAGE_TEST_NAME, SerializeToBrokenOstream) {
TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) { TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) {
std::string filename = std::string filename =
TestUtil::GetTestDataPath("net/proto2/internal/testdata/golden_message"); TestUtil::GetTestDataPath("third_party/protobuf/testdata/golden_message");
int file = open(filename.c_str(), O_RDONLY | O_BINARY); int file = open(filename.c_str(), O_RDONLY | O_BINARY);
ASSERT_GE(file, 0); ASSERT_GE(file, 0);
@ -137,7 +137,7 @@ TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) {
TEST(MESSAGE_TEST_NAME, ParsePackedFromFileDescriptor) { TEST(MESSAGE_TEST_NAME, ParsePackedFromFileDescriptor) {
std::string filename = TestUtil::GetTestDataPath( std::string filename = TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_packed_fields_message"); "third_party/protobuf/testdata/golden_packed_fields_message");
int file = open(filename.c_str(), O_RDONLY | O_BINARY); int file = open(filename.c_str(), O_RDONLY | O_BINARY);
ASSERT_GE(file, 0); ASSERT_GE(file, 0);

@ -279,7 +279,8 @@ const char* ParseContext::ReadSizeAndPushLimitAndDepth(const char* ptr,
const char* ParseContext::ParseMessage(MessageLite* msg, const char* ptr) { const char* ParseContext::ParseMessage(MessageLite* msg, const char* ptr) {
int old; int old;
ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); ptr = ReadSizeAndPushLimitAndDepth(ptr, &old);
ptr = ptr ? msg->_InternalParse(ptr, this) : nullptr; if (ptr == nullptr) return ptr;
ptr = msg->_InternalParse(ptr, this);
depth_++; depth_++;
if (!PopLimit(old)) return nullptr; if (!PopLimit(old)) return nullptr;
return ptr; return ptr;

@ -427,6 +427,17 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
bool>::type = true> bool>::type = true>
PROTOBUF_NODISCARD const char* ParseMessage(T* msg, const char* ptr); PROTOBUF_NODISCARD const char* ParseMessage(T* msg, const char* ptr);
template <typename TcParser, typename Table>
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseMessage(
MessageLite* msg, const char* ptr, const Table* table) {
int old;
ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old);
ptr = ptr ? TcParser::ParseLoop(msg, ptr, this, table) : nullptr;
depth_++;
if (!PopLimit(old)) return nullptr;
return ptr;
}
template <typename T> template <typename T>
PROTOBUF_NODISCARD PROTOBUF_NDEBUG_INLINE const char* ParseGroup( PROTOBUF_NODISCARD PROTOBUF_NDEBUG_INLINE const char* ParseGroup(
T* msg, const char* ptr, uint32_t tag) { T* msg, const char* ptr, uint32_t tag) {
@ -439,6 +450,18 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
return ptr; return ptr;
} }
template <typename TcParser, typename Table>
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseGroup(
MessageLite* msg, const char* ptr, uint32_t tag, const Table* table) {
if (--depth_ < 0) return nullptr;
group_depth_++;
ptr = TcParser::ParseLoop(msg, ptr, this, table);
group_depth_--;
depth_++;
if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr;
return ptr;
}
private: private:
// Out-of-line routine to save space in ParseContext::ParseMessage<T> // Out-of-line routine to save space in ParseContext::ParseMessage<T>
// int old; // int old;
@ -451,6 +474,11 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
PROTOBUF_NODISCARD const char* ReadSizeAndPushLimitAndDepth(const char* ptr, PROTOBUF_NODISCARD const char* ReadSizeAndPushLimitAndDepth(const char* ptr,
int* old_limit); int* old_limit);
// As above, but fully inlined for the cases where we care about performance
// more than size. eg TcParser.
PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char*
ReadSizeAndPushLimitAndDepthInlined(const char* ptr, int* old_limit);
// The context keeps an internal stack to keep track of the recursive // The context keeps an internal stack to keep track of the recursive
// part of the parse state. // part of the parse state.
// Current depth of the active parser, depth counts down. // Current depth of the active parser, depth counts down.
@ -749,12 +777,26 @@ PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg,
const char* ptr) { const char* ptr) {
int old; int old;
ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); ptr = ReadSizeAndPushLimitAndDepth(ptr, &old);
ptr = ptr ? msg->_InternalParse(ptr, this) : nullptr; if (ptr == nullptr) return ptr;
ptr = msg->_InternalParse(ptr, this);
depth_++; depth_++;
if (!PopLimit(old)) return nullptr; if (!PopLimit(old)) return nullptr;
return ptr; return ptr;
} }
inline const char* ParseContext::ReadSizeAndPushLimitAndDepthInlined(
const char* ptr, int* old_limit) {
int size = ReadSize(&ptr);
if (PROTOBUF_PREDICT_FALSE(!ptr)) {
// Make sure this isn't uninitialized even on error return
*old_limit = 0;
return nullptr;
}
*old_limit = PushLimit(ptr, size);
if (--depth_ < 0) return nullptr;
return ptr;
}
template <typename Tag, typename T> template <typename Tag, typename T>
const char* EpsCopyInputStream::ReadRepeatedFixed(const char* ptr, const char* EpsCopyInputStream::ReadRepeatedFixed(const char* ptr,
Tag expected_tag, Tag expected_tag,

@ -147,57 +147,6 @@ static const int64 kint64min = -kint64max - 1;
static const uint32 kuint32max = 0xFFFFFFFFu; static const uint32 kuint32max = 0xFFFFFFFFu;
static const uint64 kuint64max = uint64_t{0xFFFFFFFFFFFFFFFFu}; static const uint64 kuint64max = uint64_t{0xFFFFFFFFFFFFFFFFu};
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\
defined(MEMORY_SANITIZER)
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
uint16_t __sanitizer_unaligned_load16(const void *p);
uint32_t __sanitizer_unaligned_load32(const void *p);
uint64_t __sanitizer_unaligned_load64(const void *p);
void __sanitizer_unaligned_store16(void *p, uint16_t v);
void __sanitizer_unaligned_store32(void *p, uint32_t v);
void __sanitizer_unaligned_store64(void *p, uint64_t v);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) {
return __sanitizer_unaligned_load16(p);
}
inline uint32_t GOOGLE_UNALIGNED_LOAD32(const void *p) {
return __sanitizer_unaligned_load32(p);
}
inline uint64_t GOOGLE_UNALIGNED_LOAD64(const void *p) {
return __sanitizer_unaligned_load64(p);
}
inline void GOOGLE_UNALIGNED_STORE16(void *p, uint16_t v) {
__sanitizer_unaligned_store16(p, v);
}
inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32_t v) {
__sanitizer_unaligned_store32(p, v);
}
inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) {
__sanitizer_unaligned_store64(p, v);
}
#elif defined(GOOGLE_PROTOBUF_USE_UNALIGNED) && GOOGLE_PROTOBUF_USE_UNALIGNED
#define GOOGLE_UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16_t *>(_p))
#define GOOGLE_UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32_t *>(_p))
#define GOOGLE_UNALIGNED_LOAD64(_p) (*reinterpret_cast<const uint64_t *>(_p))
#define GOOGLE_UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16_t *>(_p) = (_val))
#define GOOGLE_UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32_t *>(_p) = (_val))
#define GOOGLE_UNALIGNED_STORE64(_p, _val) (*reinterpret_cast<uint64_t *>(_p) = (_val))
#else
inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) { inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) {
uint16_t t; uint16_t t;
memcpy(&t, p, sizeof t); memcpy(&t, p, sizeof t);
@ -227,7 +176,6 @@ inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32_t v) {
inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) { inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) {
memcpy(p, &v, sizeof v); memcpy(p, &v, sizeof v);
} }
#endif
#if defined(GOOGLE_PROTOBUF_OS_NACL) \ #if defined(GOOGLE_PROTOBUF_OS_NACL) \
|| (defined(__ANDROID__) && defined(__clang__) \ || (defined(__ANDROID__) && defined(__clang__) \

@ -42,11 +42,17 @@ namespace google {
namespace protobuf { namespace protobuf {
namespace TestUtil { namespace TestUtil {
// Translate net/proto2/* -> google/protobuf/* // Translate net/proto2/* or third_party/protobuf/* to google/protobuf/*.
inline std::string TranslatePathToOpensource(const std::string& google3_path) { inline std::string TranslatePathToOpensource(const std::string& google3_path) {
const std::string prefix = "net/proto2/"; std::string net_proto2 = "net/proto2/";
GOOGLE_CHECK(google3_path.find(prefix) == 0) << google3_path; std::string third_party_protobuf = "third_party/protobuf/";
std::string path = google3_path.substr(prefix.size()); std::string path;
if (google3_path.find(net_proto2) == 0) {
path = google3_path.substr(net_proto2.size());
} else {
GOOGLE_CHECK(google3_path.find(third_party_protobuf) == 0) << google3_path;
path = google3_path.substr(third_party_protobuf.size());
}
path = StringReplace(path, "internal/", "", false); path = StringReplace(path, "internal/", "", false);
path = StringReplace(path, "proto/", "", false); path = StringReplace(path, "proto/", "", false);

@ -905,16 +905,18 @@ class TextFormat::Parser::ParserImpl {
return true; return true;
} }
if (TryConsume("[")) { if (TryConsume("[")) {
while (true) { if (!TryConsume("]")) {
if (!LookingAt("{") && !LookingAt("<")) { while (true) {
DO(SkipFieldValue()); if (!LookingAt("{") && !LookingAt("<")) {
} else { DO(SkipFieldValue());
DO(SkipFieldMessage()); } else {
} DO(SkipFieldMessage());
if (TryConsume("]")) { }
break; if (TryConsume("]")) {
break;
}
DO(Consume(","));
} }
DO(Consume(","));
} }
++recursion_limit_; ++recursion_limit_;
return true; return true;

@ -94,7 +94,7 @@ class TextFormatTest : public testing::Test {
static void SetUpTestSuite() { static void SetUpTestSuite() {
GOOGLE_CHECK_OK(File::GetContents( GOOGLE_CHECK_OK(File::GetContents(
TestUtil::GetTestDataPath( TestUtil::GetTestDataPath(
"net/proto2/internal/" "third_party/protobuf/"
"testdata/text_format_unittest_data_oneof_implemented.txt"), "testdata/text_format_unittest_data_oneof_implemented.txt"),
&static_proto_text_format_, true)); &static_proto_text_format_, true));
CleanStringLineEndings(&static_proto_text_format_, false); CleanStringLineEndings(&static_proto_text_format_, false);
@ -116,7 +116,7 @@ class TextFormatExtensionsTest : public testing::Test {
public: public:
static void SetUpTestSuite() { static void SetUpTestSuite() {
GOOGLE_CHECK_OK(File::GetContents( GOOGLE_CHECK_OK(File::GetContents(
TestUtil::GetTestDataPath("net/proto2/internal/testdata/" TestUtil::GetTestDataPath("third_party/protobuf/testdata/"
"text_format_unittest_extensions_data.txt"), "text_format_unittest_extensions_data.txt"),
&static_proto_text_format_, true)); &static_proto_text_format_, true));
CleanStringLineEndings(&static_proto_text_format_, false); CleanStringLineEndings(&static_proto_text_format_, false);
@ -1035,6 +1035,19 @@ TEST_F(TextFormatTest, ParseShortRepeatedConcatenatedWithEmpty) {
EXPECT_EQ(4, proto_.repeatedgroup(1).a()); EXPECT_EQ(4, proto_.repeatedgroup(1).a());
} }
TEST_F(TextFormatTest, ParseShortRepeatedUnknownEmpty) {
std::string parse_string =
"repeated_string: \"before\"\n"
"unknown_field: []\n"
"repeated_string: \"after\"\n";
TextFormat::Parser parser;
parser.AllowUnknownField(true);
ASSERT_TRUE(parser.ParseFromString(parse_string, &proto_));
EXPECT_EQ(2, proto_.repeated_string_size());
}
TEST_F(TextFormatTest, Comments) { TEST_F(TextFormatTest, Comments) {
// Test that comments are ignored. // Test that comments are ignored.

@ -53,6 +53,11 @@ message NestedTestMessageSetContainer {
optional NestedTestMessageSetContainer child = 2; optional NestedTestMessageSetContainer child = 2;
} }
message NestedTestInt {
optional fixed32 a = 1;
optional NestedTestInt child = 2;
}
message TestMessageSetExtension1 { message TestMessageSetExtension1 {
extend proto2_wireformat_unittest.TestMessageSet { extend proto2_wireformat_unittest.TestMessageSet {
optional TestMessageSetExtension1 message_set_extension = 1545008; optional TestMessageSetExtension1 message_set_extension = 1545008;
@ -69,11 +74,6 @@ message TestMessageSetExtension2 {
optional string str = 25; optional string str = 25;
} }
message NestedTestInt {
optional fixed32 a = 1;
optional NestedTestInt child = 2;
}
message TestMessageSetExtension3 { message TestMessageSetExtension3 {
extend proto2_wireformat_unittest.TestMessageSet { extend proto2_wireformat_unittest.TestMessageSet {
optional TestMessageSetExtension3 message_set_extension = 195273129; optional TestMessageSetExtension3 message_set_extension = 195273129;

@ -31,8 +31,8 @@
#include <google/protobuf/util/internal/default_value_objectwriter.h> #include <google/protobuf/util/internal/default_value_objectwriter.h>
#include <google/protobuf/util/internal/expecting_objectwriter.h> #include <google/protobuf/util/internal/expecting_objectwriter.h>
#include <google/protobuf/util/internal/testdata/default_value_test.pb.h>
#include <google/protobuf/util/internal/constants.h> #include <google/protobuf/util/internal/constants.h>
#include <google/protobuf/util/internal/testdata/default_value_test.pb.h>
#include <google/protobuf/util/internal/type_info_test_helper.h> #include <google/protobuf/util/internal/type_info_test_helper.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>

@ -39,6 +39,10 @@
#include <google/protobuf/io/zero_copy_stream_impl_lite.h> #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor.h>
#include <google/protobuf/util/internal/expecting_objectwriter.h> #include <google/protobuf/util/internal/expecting_objectwriter.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/casts.h>
#include <google/protobuf/stubs/status.h>
#include <google/protobuf/util/internal/constants.h>
#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/anys.pb.h>
#include <google/protobuf/util/internal/testdata/books.pb.h> #include <google/protobuf/util/internal/testdata/books.pb.h>
#include <google/protobuf/util/internal/testdata/field_mask.pb.h> #include <google/protobuf/util/internal/testdata/field_mask.pb.h>
@ -46,10 +50,6 @@
#include <google/protobuf/util/internal/testdata/proto3.pb.h> #include <google/protobuf/util/internal/testdata/proto3.pb.h>
#include <google/protobuf/util/internal/testdata/struct.pb.h> #include <google/protobuf/util/internal/testdata/struct.pb.h>
#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> #include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/casts.h>
#include <google/protobuf/stubs/status.h>
#include <google/protobuf/util/internal/constants.h>
#include <google/protobuf/util/internal/type_info_test_helper.h> #include <google/protobuf/util/internal/type_info_test_helper.h>

@ -40,6 +40,12 @@
#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/message.h> #include <google/protobuf/message.h>
#include <google/protobuf/util/internal/mock_error_listener.h> #include <google/protobuf/util/internal/mock_error_listener.h>
#include <google/protobuf/stubs/bytestream.h>
#include <google/protobuf/stubs/strutil.h>
#include <gtest/gtest.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/util/internal/constants.h>
#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/anys.pb.h>
#include <google/protobuf/util/internal/testdata/books.pb.h> #include <google/protobuf/util/internal/testdata/books.pb.h>
#include <google/protobuf/util/internal/testdata/field_mask.pb.h> #include <google/protobuf/util/internal/testdata/field_mask.pb.h>
@ -49,12 +55,6 @@
#include <google/protobuf/util/internal/testdata/struct.pb.h> #include <google/protobuf/util/internal/testdata/struct.pb.h>
#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> #include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h>
#include <google/protobuf/util/internal/testdata/wrappers.pb.h> #include <google/protobuf/util/internal/testdata/wrappers.pb.h>
#include <google/protobuf/stubs/bytestream.h>
#include <google/protobuf/stubs/strutil.h>
#include <gtest/gtest.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/util/internal/constants.h>
#include <google/protobuf/util/internal/type_info_test_helper.h> #include <google/protobuf/util/internal/type_info_test_helper.h>
#include <google/protobuf/util/message_differencer.h> #include <google/protobuf/util/message_differencer.h>
#include <google/protobuf/util/type_resolver_util.h> #include <google/protobuf/util/type_resolver_util.h>

@ -34,14 +34,14 @@
#include <list> #include <list>
#include <string> #include <string>
#include <google/protobuf/util/internal/testdata/maps.pb.h>
#include <google/protobuf/util/json_format.pb.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <google/protobuf/descriptor_database.h> #include <google/protobuf/descriptor_database.h>
#include <google/protobuf/dynamic_message.h> #include <google/protobuf/dynamic_message.h>
#include <google/protobuf/io/zero_copy_stream.h> #include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/util/internal/testdata/maps.pb.h>
#include <google/protobuf/util/json_format.pb.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <google/protobuf/util/type_resolver.h> #include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/util/type_resolver_util.h> #include <google/protobuf/util/type_resolver_util.h>

@ -53,11 +53,11 @@
#include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/text_format.h> #include <google/protobuf/text_format.h>
#include <google/protobuf/wire_format.h> #include <google/protobuf/wire_format.h>
#include <google/protobuf/util/message_differencer_unittest.pb.h>
#include <google/protobuf/util/field_comparator.h> #include <google/protobuf/util/field_comparator.h>
#include <google/protobuf/util/message_differencer.h> #include <google/protobuf/util/message_differencer.h>
#include <google/protobuf/testing/googletest.h> #include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <google/protobuf/util/message_differencer_unittest.pb.h>
namespace google { namespace google {
namespace protobuf { namespace protobuf {

@ -50,19 +50,24 @@ using google::protobuf::Duration;
using google::protobuf::Timestamp; using google::protobuf::Timestamp;
namespace { namespace {
static const int kNanosPerSecond = 1000000000; static constexpr int32_t kNanosPerSecond = 1000000000;
static const int kMicrosPerSecond = 1000000; static constexpr int32_t kMicrosPerSecond = 1000000;
static const int kMillisPerSecond = 1000; static constexpr int32_t kMillisPerSecond = 1000;
static const int kNanosPerMillisecond = 1000000; static constexpr int32_t kNanosPerMillisecond = 1000000;
static const int kNanosPerMicrosecond = 1000; static constexpr int32_t kNanosPerMicrosecond = 1000;
static const int kSecondsPerMinute = 60; // Note that we ignore leap seconds. static constexpr int32_t kSecondsPerMinute =
static const int kSecondsPerHour = 3600; 60; // Note that we ignore leap seconds.
static constexpr int32_t kSecondsPerHour = 3600;
template <typename T> template <typename T>
T CreateNormalized(int64_t seconds, int64_t nanos); T CreateNormalized(int64_t seconds, int32_t nanos);
template <> template <>
Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { Timestamp CreateNormalized(int64_t seconds, int32_t nanos) {
GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds &&
seconds <= TimeUtil::kTimestampMaxSeconds)
<< "Timestamp seconds are outside of the valid range";
// Make sure nanos is in the range. // Make sure nanos is in the range.
if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) {
seconds += nanos / kNanosPerSecond; seconds += nanos / kNanosPerSecond;
@ -73,8 +78,12 @@ Timestamp CreateNormalized(int64_t seconds, int64_t nanos) {
seconds -= 1; seconds -= 1;
nanos += kNanosPerSecond; nanos += kNanosPerSecond;
} }
GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds && GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds &&
seconds <= TimeUtil::kTimestampMaxSeconds); seconds <= TimeUtil::kTimestampMaxSeconds &&
nanos >= TimeUtil::kTimestampMinNanoseconds &&
nanos <= TimeUtil::kTimestampMaxNanoseconds)
<< "Timestamp is outside of the valid range";
Timestamp result; Timestamp result;
result.set_seconds(seconds); result.set_seconds(seconds);
result.set_nanos(static_cast<int32_t>(nanos)); result.set_nanos(static_cast<int32_t>(nanos));
@ -82,7 +91,11 @@ Timestamp CreateNormalized(int64_t seconds, int64_t nanos) {
} }
template <> template <>
Duration CreateNormalized(int64_t seconds, int64_t nanos) { Duration CreateNormalized(int64_t seconds, int32_t nanos) {
GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds &&
seconds <= TimeUtil::kDurationMaxSeconds)
<< "Duration seconds are outside of the valid range";
// Make sure nanos is in the range. // Make sure nanos is in the range.
if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) {
seconds += nanos / kNanosPerSecond; seconds += nanos / kNanosPerSecond;
@ -96,8 +109,12 @@ Duration CreateNormalized(int64_t seconds, int64_t nanos) {
seconds -= 1; seconds -= 1;
nanos += kNanosPerSecond; nanos += kNanosPerSecond;
} }
GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds && GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds &&
seconds <= TimeUtil::kDurationMaxSeconds); seconds <= TimeUtil::kDurationMaxSeconds &&
nanos >= TimeUtil::kDurationMinNanoseconds &&
nanos <= TimeUtil::kDurationMaxNanoseconds)
<< "Duration is outside of the valid range";
Duration result; Duration result;
result.set_seconds(seconds); result.set_seconds(seconds);
result.set_nanos(static_cast<int32_t>(nanos)); result.set_nanos(static_cast<int32_t>(nanos));
@ -150,8 +167,12 @@ int64_t RoundTowardZero(int64_t value, int64_t divider) {
#ifndef _MSC_VER #ifndef _MSC_VER
const int64_t TimeUtil::kTimestampMinSeconds; const int64_t TimeUtil::kTimestampMinSeconds;
const int64_t TimeUtil::kTimestampMaxSeconds; const int64_t TimeUtil::kTimestampMaxSeconds;
const int32_t TimeUtil::kTimestampMinNanoseconds;
const int32_t TimeUtil::kTimestampMaxNanoseconds;
const int64_t TimeUtil::kDurationMaxSeconds; const int64_t TimeUtil::kDurationMaxSeconds;
const int64_t TimeUtil::kDurationMinSeconds; const int64_t TimeUtil::kDurationMinSeconds;
const int32_t TimeUtil::kDurationMaxNanoseconds;
const int32_t TimeUtil::kDurationMinNanoseconds;
#endif // !_MSC_VER #endif // !_MSC_VER
std::string TimeUtil::ToString(const Timestamp& timestamp) { std::string TimeUtil::ToString(const Timestamp& timestamp) {
@ -261,37 +282,43 @@ Duration TimeUtil::SecondsToDuration(int64_t seconds) {
} }
Duration TimeUtil::MinutesToDuration(int64_t minutes) { Duration TimeUtil::MinutesToDuration(int64_t minutes) {
return CreateNormalized<Duration>(minutes * kSecondsPerMinute, 0); GOOGLE_DCHECK(minutes >= TimeUtil::kDurationMinSeconds / kSecondsPerMinute &&
minutes <= TimeUtil::kDurationMaxSeconds / kSecondsPerMinute)
<< "Duration minutes are outside of the valid range";
return SecondsToDuration(minutes * kSecondsPerMinute);
} }
Duration TimeUtil::HoursToDuration(int64_t hours) { Duration TimeUtil::HoursToDuration(int64_t hours) {
return CreateNormalized<Duration>(hours * kSecondsPerHour, 0); GOOGLE_DCHECK(hours >= TimeUtil::kDurationMinSeconds / kSecondsPerHour &&
hours <= TimeUtil::kDurationMaxSeconds / kSecondsPerHour)
<< "Duration hours are outside of the valid range";
return SecondsToDuration(hours * kSecondsPerHour);
} }
int64_t TimeUtil::DurationToNanoseconds(const Duration& duration) { int64_t TimeUtil::DurationToNanoseconds(const Duration& duration) {
GOOGLE_DCHECK(IsDurationValid(duration)) << "Duration is outside of the valid range";
return duration.seconds() * kNanosPerSecond + duration.nanos(); return duration.seconds() * kNanosPerSecond + duration.nanos();
} }
int64_t TimeUtil::DurationToMicroseconds(const Duration& duration) { int64_t TimeUtil::DurationToMicroseconds(const Duration& duration) {
return duration.seconds() * kMicrosPerSecond + return RoundTowardZero(DurationToNanoseconds(duration), kNanosPerMicrosecond);
RoundTowardZero(duration.nanos(), kNanosPerMicrosecond);
} }
int64_t TimeUtil::DurationToMilliseconds(const Duration& duration) { int64_t TimeUtil::DurationToMilliseconds(const Duration& duration) {
return duration.seconds() * kMillisPerSecond + return RoundTowardZero(DurationToNanoseconds(duration), kNanosPerMillisecond);
RoundTowardZero(duration.nanos(), kNanosPerMillisecond);
} }
int64_t TimeUtil::DurationToSeconds(const Duration& duration) { int64_t TimeUtil::DurationToSeconds(const Duration& duration) {
GOOGLE_DCHECK(IsDurationValid(duration)) << "Duration is outside of the valid range";
return duration.seconds(); return duration.seconds();
} }
int64_t TimeUtil::DurationToMinutes(const Duration& duration) { int64_t TimeUtil::DurationToMinutes(const Duration& duration) {
return RoundTowardZero(duration.seconds(), kSecondsPerMinute); return RoundTowardZero(DurationToSeconds(duration), kSecondsPerMinute);
} }
int64_t TimeUtil::DurationToHours(const Duration& duration) { int64_t TimeUtil::DurationToHours(const Duration& duration) {
return RoundTowardZero(duration.seconds(), kSecondsPerHour); return RoundTowardZero(DurationToSeconds(duration), kSecondsPerHour);
} }
Timestamp TimeUtil::NanosecondsToTimestamp(int64_t nanos) { Timestamp TimeUtil::NanosecondsToTimestamp(int64_t nanos) {
@ -316,20 +343,28 @@ Timestamp TimeUtil::SecondsToTimestamp(int64_t seconds) {
} }
int64_t TimeUtil::TimestampToNanoseconds(const Timestamp& timestamp) { int64_t TimeUtil::TimestampToNanoseconds(const Timestamp& timestamp) {
GOOGLE_DCHECK(IsTimestampValid(timestamp))
<< "Timestamp is outside of the valid range";
return timestamp.seconds() * kNanosPerSecond + timestamp.nanos(); return timestamp.seconds() * kNanosPerSecond + timestamp.nanos();
} }
int64_t TimeUtil::TimestampToMicroseconds(const Timestamp& timestamp) { int64_t TimeUtil::TimestampToMicroseconds(const Timestamp& timestamp) {
GOOGLE_DCHECK(IsTimestampValid(timestamp))
<< "Timestamp is outside of the valid range";
return timestamp.seconds() * kMicrosPerSecond + return timestamp.seconds() * kMicrosPerSecond +
RoundTowardZero(timestamp.nanos(), kNanosPerMicrosecond); RoundTowardZero(timestamp.nanos(), kNanosPerMicrosecond);
} }
int64_t TimeUtil::TimestampToMilliseconds(const Timestamp& timestamp) { int64_t TimeUtil::TimestampToMilliseconds(const Timestamp& timestamp) {
GOOGLE_DCHECK(IsTimestampValid(timestamp))
<< "Timestamp is outside of the valid range";
return timestamp.seconds() * kMillisPerSecond + return timestamp.seconds() * kMillisPerSecond +
RoundTowardZero(timestamp.nanos(), kNanosPerMillisecond); RoundTowardZero(timestamp.nanos(), kNanosPerMillisecond);
} }
int64_t TimeUtil::TimestampToSeconds(const Timestamp& timestamp) { int64_t TimeUtil::TimestampToSeconds(const Timestamp& timestamp) {
GOOGLE_DCHECK(IsTimestampValid(timestamp))
<< "Timestamp is outside of the valid range";
return timestamp.seconds(); return timestamp.seconds();
} }

@ -69,11 +69,29 @@ class PROTOBUF_EXPORT TimeUtil {
// The min/max Timestamp/Duration values we support. // The min/max Timestamp/Duration values we support.
// //
// For "0001-01-01T00:00:00Z". // For "0001-01-01T00:00:00Z".
static const int64_t kTimestampMinSeconds = -62135596800LL; static constexpr int64_t kTimestampMinSeconds = -62135596800LL;
// For "9999-12-31T23:59:59.999999999Z". // For "9999-12-31T23:59:59.999999999Z".
static const int64_t kTimestampMaxSeconds = 253402300799LL; static constexpr int64_t kTimestampMaxSeconds = 253402300799LL;
static const int64_t kDurationMinSeconds = -315576000000LL; static constexpr int32_t kTimestampMinNanoseconds = 0;
static const int64_t kDurationMaxSeconds = 315576000000LL; static constexpr int32_t kTimestampMaxNanoseconds = 999999999;
static constexpr int64_t kDurationMinSeconds = -315576000000LL;
static constexpr int64_t kDurationMaxSeconds = 315576000000LL;
static constexpr int32_t kDurationMinNanoseconds = -999999999;
static constexpr int32_t kDurationMaxNanoseconds = 999999999;
static bool IsTimestampValid(const Timestamp& timestamp) {
return timestamp.seconds() <= kTimestampMaxSeconds &&
timestamp.seconds() >= kTimestampMinSeconds &&
timestamp.nanos() <= kTimestampMaxNanoseconds &&
timestamp.nanos() >= kTimestampMinNanoseconds;
}
static bool IsDurationValid(const Duration& duration) {
return duration.seconds() <= kDurationMaxSeconds &&
duration.seconds() >= kDurationMinSeconds &&
duration.nanos() <= kDurationMaxNanoseconds &&
duration.nanos() >= kDurationMinNanoseconds;
}
// Converts Timestamp to/from RFC 3339 date string format. // Converts Timestamp to/from RFC 3339 date string format.
// Generated output will always be Z-normalized and uses 3, 6 or 9 // Generated output will always be Z-normalized and uses 3, 6 or 9

@ -378,6 +378,138 @@ TEST(TimeUtilTest, TimestampOperators) {
EXPECT_TRUE(t2 != t1); EXPECT_TRUE(t2 != t1);
} }
TEST(TimeUtilTest, IsDurationValid) {
Duration valid;
Duration overflow;
overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1);
Duration underflow;
underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1);
Duration overflow_nanos;
overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1);
Duration underflow_nanos;
underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1);
EXPECT_TRUE(TimeUtil::IsDurationValid(valid));
EXPECT_FALSE(TimeUtil::IsDurationValid(overflow));
EXPECT_FALSE(TimeUtil::IsDurationValid(underflow));
EXPECT_FALSE(TimeUtil::IsDurationValid(overflow_nanos));
EXPECT_FALSE(TimeUtil::IsDurationValid(underflow_nanos));
}
TEST(TimeUtilTest, IsTimestampValid) {
Timestamp valid;
Timestamp overflow;
overflow.set_seconds(TimeUtil::kTimestampMaxSeconds + 1);
Timestamp underflow;
underflow.set_seconds(TimeUtil::kTimestampMinSeconds - 1);
Timestamp overflow_nanos;
overflow_nanos.set_nanos(TimeUtil::kTimestampMaxNanoseconds + 1);
Timestamp underflow_nanos;
underflow_nanos.set_nanos(TimeUtil::kTimestampMinNanoseconds - 1);
EXPECT_TRUE(TimeUtil::IsTimestampValid(valid));
EXPECT_FALSE(TimeUtil::IsTimestampValid(overflow));
EXPECT_FALSE(TimeUtil::IsTimestampValid(underflow));
EXPECT_FALSE(TimeUtil::IsTimestampValid(overflow_nanos));
EXPECT_FALSE(TimeUtil::IsTimestampValid(underflow_nanos));
}
#ifdef PROTOBUF_HAS_DEATH_TEST // death tests do not work on Windows yet.
TEST(TimeUtilTest, DurationBounds) {
Duration overflow;
overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1);
Duration underflow;
underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1);
Duration overflow_nanos;
overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1);
Duration underflow_nanos;
underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1);
EXPECT_DEBUG_DEATH({ TimeUtil::SecondsToDuration(overflow.seconds()); },
"Duration seconds");
EXPECT_DEBUG_DEATH({ TimeUtil::SecondsToDuration(underflow.seconds()); },
"Duration seconds");
EXPECT_DEBUG_DEATH(
{ TimeUtil::MinutesToDuration(overflow.seconds() / 60 + 1); },
"Duration minutes");
EXPECT_DEBUG_DEATH(
{ TimeUtil::MinutesToDuration(underflow.seconds() / 60 - 1); },
"Duration minutes");
EXPECT_DEBUG_DEATH(
{ TimeUtil::HoursToDuration(overflow.seconds() / 60 + 1); },
"Duration hours");
EXPECT_DEBUG_DEATH(
{ TimeUtil::HoursToDuration(underflow.seconds() / 60 - 1); },
"Duration hours");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(overflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(underflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(overflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(underflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(overflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(underflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(overflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(underflow_nanos); },
"outside of the valid range");
}
TEST(TimeUtilTest, TimestampBounds) {
Timestamp overflow;
overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1);
Timestamp underflow;
underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1);
Timestamp overflow_nanos;
overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1);
Timestamp underflow_nanos;
underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1);
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(overflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(underflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(overflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(underflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(overflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(underflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(overflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(underflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(overflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(underflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(overflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(underflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(overflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(underflow); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(overflow_nanos); },
"outside of the valid range");
EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(underflow_nanos); },
"outside of the valid range");
}
#endif // PROTOBUF_HAS_DEATH_TEST
} // namespace } // namespace
} // namespace util } // namespace util
} // namespace protobuf } // namespace protobuf

@ -42,10 +42,10 @@
#include <google/protobuf/test_util.h> #include <google/protobuf/test_util.h>
#include <google/protobuf/unittest.pb.h> #include <google/protobuf/unittest.pb.h>
#include <google/protobuf/unittest_custom_options.pb.h> #include <google/protobuf/unittest_custom_options.pb.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <google/protobuf/util/type_resolver.h> #include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/testing/googletest.h> #include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
namespace google { namespace google {
namespace protobuf { namespace protobuf {

Loading…
Cancel
Save