Merge branch 'sync-piper' into sync-stage

pull/7435/head
Joshua Haberman 5 years ago
commit 140e1bb70e
  1. 61
      conformance/binary_json_conformance_suite.h
  2. 48
      conformance/conformance_test.h
  3. 18
      conformance/text_format_conformance_suite.h
  4. 45
      java/core/src/main/java/com/google/protobuf/Descriptors.java
  5. 41
      python/google/protobuf/internal/text_format_test.py
  6. 20
      python/google/protobuf/internal/type_checkers.py
  7. 21
      python/google/protobuf/json_format.py
  8. 14
      python/google/protobuf/text_format.py
  9. 3
      src/google/protobuf/arena_unittest.cc
  10. 1
      src/google/protobuf/compiler/cpp/cpp_map_field.cc
  11. 6
      src/google/protobuf/compiler/java/java_enum.cc
  12. 6
      src/google/protobuf/compiler/java/java_enum_lite.cc
  13. 1
      src/google/protobuf/compiler/java/java_field.cc
  14. 1
      src/google/protobuf/compiler/java/java_helpers.cc
  15. 1
      src/google/protobuf/compiler/java/java_helpers.h
  16. 1
      src/google/protobuf/compiler/java/java_primitive_field.cc
  17. 4
      src/google/protobuf/compiler/java/java_primitive_field_lite.cc
  18. 16
      src/google/protobuf/descriptor.cc
  19. 12
      src/google/protobuf/descriptor.h
  20. 20
      src/google/protobuf/descriptor_unittest.cc
  21. 6
      src/google/protobuf/extension_set_unittest.cc
  22. 2
      src/google/protobuf/implicit_weak_message.h
  23. 2
      src/google/protobuf/io/coded_stream_unittest.cc
  24. 6
      src/google/protobuf/map_entry.h
  25. 17
      src/google/protobuf/map_field.cc
  26. 2
      src/google/protobuf/map_field.h
  27. 4
      src/google/protobuf/map_field_test.cc
  28. 45
      src/google/protobuf/map_test_util.h
  29. 6
      src/google/protobuf/map_type_handler.h
  30. 3
      src/google/protobuf/message.h
  31. 3
      src/google/protobuf/message_lite.h
  32. 4
      src/google/protobuf/metadata_lite.h
  33. 11
      src/google/protobuf/reflection.h
  34. 2
      src/google/protobuf/repeated_field.cc
  35. 145
      src/google/protobuf/repeated_field.h
  36. 67
      src/google/protobuf/repeated_field_unittest.cc
  37. 5
      src/google/protobuf/util/internal/datapiece.cc
  38. 14
      src/google/protobuf/util/internal/json_escaping.h
  39. 5
      src/google/protobuf/util/internal/protostream_objectsource.cc
  40. 22
      src/google/protobuf/util/time_util_test.cc
  41. 24
      src/google/protobuf/wire_format_lite.h

@ -53,35 +53,34 @@ class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
void RunJsonTestsForStruct();
void RunJsonTestsForValue();
void RunJsonTestsForAny();
void RunValidJsonTest(const string& test_name,
ConformanceLevel level,
const string& input_json,
const string& equivalent_text_format);
void RunValidJsonTest(const std::string& test_name, ConformanceLevel level,
const std::string& input_json,
const std::string& equivalent_text_format);
void RunValidJsonTestWithProtobufInput(
const string& test_name,
ConformanceLevel level,
const std::string& test_name, ConformanceLevel level,
const protobuf_test_messages::proto3::TestAllTypesProto3& input,
const string& equivalent_text_format);
void RunValidJsonIgnoreUnknownTest(
const string& test_name, ConformanceLevel level, const string& input_json,
const string& equivalent_text_format);
void RunValidProtobufTest(const string& test_name, ConformanceLevel level,
const string& input_protobuf,
const string& equivalent_text_format,
const std::string& equivalent_text_format);
void RunValidJsonIgnoreUnknownTest(const std::string& test_name,
ConformanceLevel level,
const std::string& input_json,
const std::string& equivalent_text_format);
void RunValidProtobufTest(const std::string& test_name,
ConformanceLevel level,
const std::string& input_protobuf,
const std::string& equivalent_text_format,
bool is_proto3);
void RunValidBinaryProtobufTest(const string& test_name,
void RunValidBinaryProtobufTest(const std::string& test_name,
ConformanceLevel level,
const string& input_protobuf,
const std::string& input_protobuf,
bool is_proto3);
void RunValidBinaryProtobufTest(const string& test_name,
void RunValidBinaryProtobufTest(const std::string& test_name,
ConformanceLevel level,
const string& input_protobuf,
const string& expected_protobuf,
const std::string& input_protobuf,
const std::string& expected_protobuf,
bool is_proto3);
void RunValidProtobufTestWithMessage(
const string& test_name, ConformanceLevel level,
const Message *input,
const string& equivalent_text_format,
const std::string& test_name, ConformanceLevel level,
const Message* input, const std::string& equivalent_text_format,
bool is_proto3);
bool ParseJsonResponse(
@ -93,20 +92,20 @@ class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
Message* test_message) override;
typedef std::function<bool(const Json::Value&)> Validator;
void RunValidJsonTestWithValidator(const string& test_name,
void RunValidJsonTestWithValidator(const std::string& test_name,
ConformanceLevel level,
const string& input_json,
const std::string& input_json,
const Validator& validator);
void ExpectParseFailureForJson(const string& test_name,
void ExpectParseFailureForJson(const std::string& test_name,
ConformanceLevel level,
const string& input_json);
void ExpectSerializeFailureForJson(const string& test_name,
const std::string& input_json);
void ExpectSerializeFailureForJson(const std::string& test_name,
ConformanceLevel level,
const string& text_format);
void ExpectParseFailureForProtoWithProtoVersion (const string& proto,
const string& test_name,
ConformanceLevel level,
bool is_proto3);
const std::string& text_format);
void ExpectParseFailureForProtoWithProtoVersion(const std::string& proto,
const std::string& test_name,
ConformanceLevel level,
bool is_proto3);
void ExpectParseFailureForProto(const std::string& proto,
const std::string& test_name,
ConformanceLevel level);

@ -88,7 +88,7 @@ class ForkPipeRunner : public ConformanceTestRunner {
const std::vector<ConformanceTestSuite*>& suites);
ForkPipeRunner(const std::string& executable,
const std::vector<string>& executable_args)
const std::vector<std::string>& executable_args)
: child_pid_(-1),
executable_(executable),
executable_args_(executable_args) {}
@ -113,7 +113,7 @@ class ForkPipeRunner : public ConformanceTestRunner {
int read_fd_;
pid_t child_pid_;
std::string executable_;
const std::vector<string> executable_args_;
const std::vector<std::string> executable_args_;
std::string current_test_name_;
};
@ -168,9 +168,7 @@ class ConformanceTestSuite {
// Gets the flag name to the failure list file.
// By default, this would return --failure_list
string GetFailureListFlagName() {
return failure_list_flag_name_;
}
std::string GetFailureListFlagName() { return failure_list_flag_name_; }
void SetFailureListFlagName(const std::string& failure_list_flag_name) {
failure_list_flag_name_ = failure_list_flag_name;
@ -207,18 +205,18 @@ class ConformanceTestSuite {
class ConformanceRequestSetting {
public:
ConformanceRequestSetting(
ConformanceLevel level,
conformance::WireFormat input_format,
conformance::WireFormat output_format,
conformance::TestCategory test_category,
const Message& prototype_message,
const string& test_name, const string& input);
ConformanceRequestSetting(ConformanceLevel level,
conformance::WireFormat input_format,
conformance::WireFormat output_format,
conformance::TestCategory test_category,
const Message& prototype_message,
const std::string& test_name,
const std::string& input);
virtual ~ConformanceRequestSetting() {}
std::unique_ptr<Message> NewTestMessage() const;
string GetTestName() const;
std::string GetTestName() const;
const conformance::ConformanceRequest& GetRequest() const {
return request_;
@ -228,7 +226,7 @@ class ConformanceTestSuite {
return level_;
}
string ConformanceLevelToString(ConformanceLevel level) const;
std::string ConformanceLevelToString(ConformanceLevel level) const;
void SetPrintUnknownFields(bool print_unknown_fields) {
request_.set_print_unknown_fields(true);
@ -239,8 +237,9 @@ class ConformanceTestSuite {
}
protected:
virtual string InputFormatString(conformance::WireFormat format) const;
virtual string OutputFormatString(conformance::WireFormat format) const;
virtual std::string InputFormatString(conformance::WireFormat format) const;
virtual std::string OutputFormatString(
conformance::WireFormat format) const;
conformance::ConformanceRequest request_;
private:
@ -249,12 +248,12 @@ class ConformanceTestSuite {
::conformance::WireFormat output_format_;
const Message& prototype_message_;
std::unique_ptr<Message> prototype_message_for_compare_;
string test_name_;
std::string test_name_;
};
bool CheckSetEmpty(const std::set<string>& set_to_check,
bool CheckSetEmpty(const std::set<std::string>& set_to_check,
const std::string& write_to_file, const std::string& msg);
string WireFormatToString(conformance::WireFormat wire_format);
std::string WireFormatToString(conformance::WireFormat wire_format);
// Parse payload in the response to the given message. Returns true on
// success.
@ -264,24 +263,23 @@ class ConformanceTestSuite {
Message* test_message) = 0;
void VerifyResponse(const ConformanceRequestSetting& setting,
const string& equivalent_wire_format,
const std::string& equivalent_wire_format,
const conformance::ConformanceResponse& response,
bool need_report_success, bool require_same_wire_format);
void ReportSuccess(const std::string& test_name);
void ReportFailure(const string& test_name,
ConformanceLevel level,
void ReportFailure(const std::string& test_name, ConformanceLevel level,
const conformance::ConformanceRequest& request,
const conformance::ConformanceResponse& response,
const char* fmt, ...);
void ReportSkip(const string& test_name,
void ReportSkip(const std::string& test_name,
const conformance::ConformanceRequest& request,
const conformance::ConformanceResponse& response);
void RunValidInputTest(const ConformanceRequestSetting& setting,
const string& equivalent_text_format);
const std::string& equivalent_text_format);
void RunValidBinaryInputTest(const ConformanceRequestSetting& setting,
const string& equivalent_wire_format,
const std::string& equivalent_wire_format,
bool require_same_wire_format = false);
void RunTest(const std::string& test_name,

@ -42,19 +42,19 @@ class TextFormatConformanceTestSuite : public ConformanceTestSuite {
private:
void RunSuiteImpl();
void RunValidTextFormatTest(const string& test_name, ConformanceLevel level,
const string& input);
void RunValidTextFormatTestProto2(const string& test_name,
void RunValidTextFormatTest(const std::string& test_name,
ConformanceLevel level, const std::string& input);
void RunValidTextFormatTestProto2(const std::string& test_name,
ConformanceLevel level,
const string& input);
void RunValidTextFormatTestWithMessage(const string& test_name,
const std::string& input);
void RunValidTextFormatTestWithMessage(const std::string& test_name,
ConformanceLevel level,
const string& input_text,
const std::string& input_text,
const Message& prototype);
void RunValidUnknownTextFormatTest(const string& test_name,
void RunValidUnknownTextFormatTest(const std::string& test_name,
const Message& message);
void ExpectParseFailure(const string& test_name, ConformanceLevel level,
const string& input);
void ExpectParseFailure(const std::string& test_name, ConformanceLevel level,
const std::string& input);
bool ParseTextFormatResponse(const conformance::ConformanceResponse& response,
const ConformanceRequestSetting& setting,
Message* test_message);

@ -702,6 +702,11 @@ public final class Descriptors {
return Collections.unmodifiableList(Arrays.asList(oneofs));
}
/** Get a list of this message type's real oneofs. */
public List<OneofDescriptor> getRealOneofs() {
return Collections.unmodifiableList(Arrays.asList(oneofs).subList(0, realOneofCount));
}
/** Get a list of this message type's extensions. */
public List<FieldDescriptor> getExtensions() {
return Collections.unmodifiableList(Arrays.asList(extensions));
@ -821,6 +826,7 @@ public final class Descriptors {
private final FieldDescriptor[] fields;
private final FieldDescriptor[] extensions;
private final OneofDescriptor[] oneofs;
private final int realOneofCount;
// Used to create a placeholder when the type cannot be found.
Descriptor(final String fullname) throws DescriptorValidationException {
@ -846,6 +852,7 @@ public final class Descriptors {
this.fields = new FieldDescriptor[0];
this.extensions = new FieldDescriptor[0];
this.oneofs = new OneofDescriptor[0];
this.realOneofCount = 0;
// Create a placeholder FileDescriptor to hold this message.
this.file = new FileDescriptor(packageName, this);
@ -899,6 +906,18 @@ public final class Descriptors {
}
}
int syntheticOneofCount = 0;
for (OneofDescriptor oneof : this.oneofs) {
if (oneof.isSynthetic()) {
syntheticOneofCount++;
} else {
if (syntheticOneofCount > 0) {
throw new DescriptorValidationException(this, "Synthetic oneofs must come last.");
}
}
}
this.realOneofCount = this.oneofs.length - syntheticOneofCount;
file.pool.addSymbol(this);
}
@ -1125,6 +1144,11 @@ public final class Descriptors {
return containingOneof;
}
/** Get the field's containing oneof, only if non-synthetic. */
public OneofDescriptor getRealContainingOneof() {
return containingOneof != null && !containingOneof.isSynthetic() ? containingOneof : null;
}
/**
* Returns true if this field was syntactically written with "optional" in the .proto file.
* Excludes singular proto3 fields that do not have a label.
@ -1135,22 +1159,23 @@ public final class Descriptors {
}
/**
* Returns true if this is a non-oneof field that tracks presence.
* Returns true if this field tracks presence, ie. does the field distinguish between "unset"
* and "present with default value."
*
* <p>This includes all "required" and "optional" fields in the .proto file, but excludes oneof
* fields and singular proto3 fields without "optional".
* <p>This includes required, optional, and oneof fields. It excludes maps, repeated fields, and
* singular proto3 fields without "optional".
*
* <p>In implementations that use hasbits, this method will probably indicate whether this field
* uses a hasbit.
* <p>For fields where hasPresence() == true, the return value of msg.hasField() is semantically
* meaningful.
*/
boolean isSingularWithPresence() {
boolean hasPresence() {
if (isRepeated()) {
return false;
}
if (getContainingOneof() != null && !getContainingOneof().isSynthetic()) {
return false;
}
return getType() == Type.MESSAGE || isProto3Optional || file.getSyntax() == Syntax.PROTO2;
return getType() == Type.MESSAGE
|| getType() == Type.GROUP
|| getContainingOneof() != null
|| file.getSyntax() == Syntax.PROTO2;
}
/**

@ -57,6 +57,7 @@ from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf import descriptor_pb2
from google.protobuf.internal import any_test_pb2 as test_extend_any
from google.protobuf.internal import message_set_extensions_pb2
from google.protobuf.internal import test_proto3_optional_pb2
from google.protobuf.internal import test_util
from google.protobuf import descriptor_pool
from google.protobuf import text_format
@ -176,14 +177,12 @@ class TextFormatMessageToStringTests(TextFormatBase):
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'repeated_float: 0\n'
# This should be 0.8
'repeated_float: 0.80000001\n'
'repeated_float: 0.8\n'
'repeated_float: 1\n'
'repeated_float: 1.2\n'
'repeated_float: 1.23\n'
'repeated_float: 1.234\n'
# This should be 1.2345
'repeated_float: 1.2345001\n'
'repeated_float: 1.2345\n'
'repeated_float: 1.23456\n'
# Note that these don't use scientific notation.
'repeated_float: 12000000000\n'
@ -400,8 +399,7 @@ class TextFormatMessageToStringTests(TextFormatBase):
# 32-bit 1.2 is noisy when extended to 64-bit:
# >>> struct.unpack('f', struct.pack('f', 1.2))[0]
# 1.2000000476837158
# TODO(jieluo): change to 1.2 with cl/241634942.
message.payload.optional_float = 1.2000000476837158
message.payload.optional_float = 1.2
formatted_fields = ['optional_float: 1.2',
'optional_double: -3.45678901234568e-6',
'repeated_float: -5642', 'repeated_double: 7.89e-5']
@ -422,7 +420,7 @@ class TextFormatMessageToStringTests(TextFormatBase):
'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format(
*formatted_fields))
# Test default float_format has 8 valid digits.
# Test default float_format will automatic print shortest float.
message.payload.optional_float = 1.2345678912
message.payload.optional_double = 1.2345678912
formatted_fields = ['optional_float: 1.2345679',
@ -434,6 +432,17 @@ class TextFormatMessageToStringTests(TextFormatBase):
'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format(
*formatted_fields))
message.Clear()
message.payload.optional_float = 1.1000000000011
self.assertEqual(text_format.MessageToString(message),
'payload {\n optional_float: 1.1\n}\n')
message.payload.optional_float = 1.00000075e-36
self.assertEqual(text_format.MessageToString(message),
'payload {\n optional_float: 1.00000075e-36\n}\n')
message.payload.optional_float = 12345678912345e+11
self.assertEqual(text_format.MessageToString(message),
'payload {\n optional_float: 1.234568e+24\n}\n')
def testMessageToString(self, message_module):
message = message_module.ForeignMessage()
message.c = 123
@ -1850,6 +1859,24 @@ class Proto3Tests(unittest.TestCase):
text_format.Merge(text, message)
self.assertEqual(str(e.exception), '3:11 : Expected "}".')
def testProto3Optional(self):
msg = test_proto3_optional_pb2.TestProto3Optional()
self.assertEqual(text_format.MessageToString(msg), '')
msg.optional_int32 = 0
msg.optional_float = 0.0
msg.optional_string = ''
msg.optional_nested_message.bb = 0
text = ('optional_int32: 0\n'
'optional_float: 0.0\n'
'optional_string: ""\n'
'optional_nested_message {\n'
' bb: 0\n'
'}\n')
self.assertEqual(text_format.MessageToString(msg), text)
msg2 = test_proto3_optional_pb2.TestProto3Optional()
text_format.Parse(text, msg2)
self.assertEqual(text_format.MessageToString(msg2), text)
class TokenizerTest(unittest.TestCase):

@ -72,6 +72,26 @@ def TruncateToFourByteFloat(original):
return struct.unpack('<f', struct.pack('<f', original))[0]
def ToShortestFloat(original):
"""Returns the shortest float that has same value in wire."""
# Return the original value if it is not truncated. This may happen
# if someone mixes this code with an old protobuf runtime.
# TODO(jieluo): Remove it after maybe 2022.
if TruncateToFourByteFloat(original) != original:
return original
# All 4 byte floats have between 6 and 9 significant digits, so we
# start with 6 as the lower bound.
# It has to be iterative because use '.9g' directly can not get rid
# of the noises for most values. For example if set a float_field=0.9
# use '.9g' will print 0.899999976.
precision = 6
rounded = float('{0:.{1}g}'.format(original, precision))
while TruncateToFourByteFloat(rounded) != original:
precision += 1
rounded = float('{0:.{1}g}'.format(original, precision))
return rounded
def SupportsOpenEnums(field_descriptor):
return field_descriptor.containing_type.syntax == "proto3"

@ -283,25 +283,6 @@ class _Printer(object):
def _FieldToJsonObject(self, field, value):
"""Converts field value according to Proto3 JSON Specification."""
def _ToShortestFloat(original):
"""Returns the shortest float that has same value in wire."""
# Return the original value if it is not truncated. This may happen
# if someone mixes this code with an old protobuf runtime.
if type_checkers.TruncateToFourByteFloat(original) != original:
return original
# All 4 byte floats have between 6 and 9 significant digits, so we
# start with 6 as the lower bound.
# It has to be iterative because use '.9g' directly can not get rid
# of the noises for most values. For example if set a float_field=0.9
# use '.9g' will print 0.899999976.
precision = 6
rounded = float('{0:.{1}g}'.format(original, precision))
while type_checkers.TruncateToFourByteFloat(rounded) != original:
precision += 1
rounded = float('{0:.{1}g}'.format(original, precision))
return rounded
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
return self._MessageToJsonObject(value)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
@ -337,7 +318,7 @@ class _Printer(object):
if self.float_format:
return float(format(value, self.float_format))
else:
return _ToShortestFloat(value)
return type_checkers.ToShortestFloat(value)
return value

@ -158,9 +158,9 @@ def MessageToString(
determined by the extension number. By default, use the field number
order.
float_format (str): If set, use this to specify float field formatting
(per the "Format Specification Mini-Language"); otherwise, 8 valid
digits is used (default '.8g'). Also affect double field if
double_format is not set but float_format is set.
(per the "Format Specification Mini-Language"); otherwise, shortest float
that has same value in wire will be printed. Also affect double field
if double_format is not set but float_format is set.
double_format (str): If set, use this to specify double field formatting
(per the "Format Specification Mini-Language"); if it is not set but
float_format is set, use float_format. Otherwise, use ``str()``
@ -367,9 +367,9 @@ class _Printer(object):
defined in source code instead of the field number. By default, use the
field number order.
float_format: If set, use this to specify float field formatting
(per the "Format Specification Mini-Language"); otherwise, 8 valid
digits is used (default '.8g'). Also affect double field if
double_format is not set but float_format is set.
(per the "Format Specification Mini-Language"); otherwise, shortest
float that has same value in wire will be printed. Also affect double
field if double_format is not set but float_format is set.
double_format: If set, use this to specify double field formatting
(per the "Format Specification Mini-Language"); if it is not set but
float_format is set, use float_format. Otherwise, str() is used.
@ -630,7 +630,7 @@ class _Printer(object):
if self.float_format is not None:
out.write('{1:{0}}'.format(self.float_format, value))
else:
out.write(str(float(format(value, '.8g'))))
out.write(str(type_checkers.ToShortestFloat(value)))
elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_DOUBLE and
self.double_format is not None):
out.write('{1:{0}}'.format(self.double_format, value))

@ -58,6 +58,9 @@
#include <google/protobuf/stubs/strutil.h>
// Must be included last
#include <google/protobuf/port_def.inc>
using proto2_arena_unittest::ArenaMessage;
using protobuf_unittest::TestAllExtensions;
using protobuf_unittest::TestAllTypes;

@ -29,6 +29,7 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <google/protobuf/compiler/cpp/cpp_map_field.h>
#include <google/protobuf/compiler/cpp/cpp_helpers.h>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/wire_format.h>

@ -142,9 +142,13 @@ void EnumGenerator::Generate(io::Printer* printer) {
vars["number"] = StrCat(descriptor_->value(i)->number());
vars["{"] = "";
vars["}"] = "";
vars["deprecation"] = descriptor_->value(i)->options().deprecated()
? "@java.lang.Deprecated "
: "";
WriteEnumValueDocComment(printer, descriptor_->value(i));
printer->Print(vars,
"public static final int ${$$name$_VALUE$}$ = $number$;\n");
"$deprecation$public static final int ${$$name$_VALUE$}$ = "
"$number$;\n");
printer->Annotate("{", "}", descriptor_->value(i));
}
printer->Print("\n");

@ -124,9 +124,13 @@ void EnumLiteGenerator::Generate(io::Printer* printer) {
vars["number"] = StrCat(descriptor_->value(i)->number());
vars["{"] = "";
vars["}"] = "";
vars["deprecation"] = descriptor_->value(i)->options().deprecated()
? "@java.lang.Deprecated "
: "";
WriteEnumValueDocComment(printer, descriptor_->value(i));
printer->Print(vars,
"public static final int ${$$name$_VALUE$}$ = $number$;\n");
"$deprecation$public static final int ${$$name$_VALUE$}$ = "
"$number$;\n");
printer->Annotate("{", "}", descriptor_->value(i));
}
printer->Print("\n");

@ -251,6 +251,7 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor,
(*variables)["disambiguated_reason"] = info->disambiguated_reason;
(*variables)["constant_name"] = FieldConstantName(descriptor);
(*variables)["number"] = StrCat(descriptor->number());
(*variables)["kt_dsl_builder"] = "_builder";
// These variables are placeholders to pick out the beginning and ends of
// identifiers for annotations (when doing so with existing variables would
// be ambiguous or impossible). They should never be set to anything but the

@ -428,6 +428,7 @@ const char* BoxedPrimitiveTypeName(const FieldDescriptor* descriptor) {
return BoxedPrimitiveTypeName(GetJavaType(descriptor));
}
std::string GetOneofStoredType(const FieldDescriptor* field) {
const JavaType javaType = GetJavaType(field);
switch (javaType) {

@ -226,6 +226,7 @@ const char* PrimitiveTypeName(JavaType type);
// types.
const char* BoxedPrimitiveTypeName(JavaType type);
// Get the name of the java enum constant representing this type. E.g.,
// "INT32" for FieldDescriptor::TYPE_INT32. The enum constant's full
// name is "com.google.protobuf.WireFormat.FieldType.INT32".

@ -296,6 +296,7 @@ void ImmutablePrimitiveFieldGenerator::GenerateBuilderMembers(
"}\n");
}
void ImmutablePrimitiveFieldGenerator::GenerateFieldBuilderInitializationCode(
io::Printer* printer) const {
// noop for primitives

@ -32,6 +32,8 @@
// Based on original Protocol Buffers design by
// Sanjay Ghemawat, Jeff Dean, and others.
#include <google/protobuf/compiler/java/java_primitive_field_lite.h>
#include <map>
#include <string>
@ -41,7 +43,6 @@
#include <google/protobuf/compiler/java/java_doc_comment.h>
#include <google/protobuf/compiler/java/java_helpers.h>
#include <google/protobuf/compiler/java/java_name_resolver.h>
#include <google/protobuf/compiler/java/java_primitive_field_lite.h>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/wire_format.h>
#include <google/protobuf/stubs/strutil.h>
@ -300,6 +301,7 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers(
printer->Annotate("{", "}", descriptor_);
}
void ImmutablePrimitiveFieldLiteGenerator::GenerateFieldInfo(
io::Printer* printer, std::vector<uint16>* output) const {
WriteIntToUtf16CharSequence(descriptor_->number(), output);

@ -1720,6 +1720,18 @@ const EnumValueDescriptor* Descriptor::FindEnumValueByName(
}
}
const FieldDescriptor* Descriptor::map_key() const {
if (!options().map_entry()) return nullptr;
GOOGLE_DCHECK_EQ(field_count(), 2);
return field(0);
}
const FieldDescriptor* Descriptor::map_value() const {
if (!options().map_entry()) return nullptr;
GOOGLE_DCHECK_EQ(field_count(), 2);
return field(1);
}
const EnumValueDescriptor* EnumDescriptor::FindValueByName(
const std::string& key) const {
Symbol result =
@ -6222,8 +6234,8 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field,
return false;
}
const FieldDescriptor* key = message->field(0);
const FieldDescriptor* value = message->field(1);
const FieldDescriptor* key = message->map_key();
const FieldDescriptor* value = message->map_value();
if (key->label() != FieldDescriptor::LABEL_OPTIONAL || key->number() != 1 ||
key->name() != "key") {
return false;

@ -487,6 +487,16 @@ class PROTOBUF_EXPORT Descriptor {
// |*out_location| unchanged iff location information was not available.
bool GetSourceLocation(SourceLocation* out_location) const;
// Maps --------------------------------------------------------------
// Returns the FieldDescriptor for the "key" field. If this isn't a map entry
// field, returns nullptr.
const FieldDescriptor* map_key() const;
// Returns the FieldDescriptor for the "value" field. If this isn't a map
// entry field, returns nullptr.
const FieldDescriptor* map_value() const;
private:
typedef MessageOptions OptionsType;
@ -693,7 +703,7 @@ class PROTOBUF_EXPORT FieldDescriptor {
// .proto file. Excludes singular proto3 fields that do not have a label.
bool has_optional_keyword() const;
// Returns true if this field tracks presence, ie. does the message
// Returns true if this field tracks presence, ie. does the field
// distinguish between "unset" and "present with default value."
// This includes required, optional, and oneof fields. It excludes maps,
// repeated fields, and singular proto3 fields without "optional".

@ -999,6 +999,22 @@ TEST_F(DescriptorTest, IsMap) {
EXPECT_TRUE(map_->message_type()->options().map_entry());
}
TEST_F(DescriptorTest, GetMap) {
const Descriptor* map_desc = map_->message_type();
const FieldDescriptor* map_key = map_desc->map_key();
ASSERT_TRUE(map_key != nullptr);
EXPECT_EQ(map_key->name(), "key");
EXPECT_EQ(map_key->number(), 1);
const FieldDescriptor* map_value = map_desc->map_value();
ASSERT_TRUE(map_value != nullptr);
EXPECT_EQ(map_value->name(), "value");
EXPECT_EQ(map_value->number(), 2);
EXPECT_EQ(message_->map_key(), nullptr);
EXPECT_EQ(message_->map_value(), nullptr);
}
TEST_F(DescriptorTest, FieldHasDefault) {
EXPECT_FALSE(foo_->has_default_value());
EXPECT_FALSE(bar_->has_default_value());
@ -7184,9 +7200,9 @@ class SourceLocationTest : public testing::Test {
DescriptorPool pool_;
// tag number of all custom options in above test file
static const int kCustomOptionFieldNumber = 10101;
static constexpr int kCustomOptionFieldNumber = 10101;
// tag number of field "a" in message type "A" in above test file
static const int kAFieldNumber = 1;
static constexpr int kAFieldNumber = 1;
};
// TODO(adonovan): implement support for option fields and for

@ -821,7 +821,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
const int old_capacity = \
message.GetRepeatedExtension(unittest::repeated_##type##_extension) \
.Capacity(); \
EXPECT_GE(old_capacity, kMinRepeatedFieldAllocationSize); \
EXPECT_GE(old_capacity, kRepeatedFieldLowerClampLimit); \
for (int i = 0; i < 16; ++i) { \
message.AddExtension(unittest::repeated_##type##_extension, value); \
} \
@ -864,7 +864,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
message.AddExtension(unittest::repeated_string_extension, value);
}
min_expected_size +=
(sizeof(value) + value.size()) * (16 - kMinRepeatedFieldAllocationSize);
(sizeof(value) + value.size()) * (16 - kRepeatedFieldLowerClampLimit);
EXPECT_LE(min_expected_size, message.SpaceUsed());
}
// Repeated messages
@ -880,7 +880,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
->CopyFrom(prototype);
}
min_expected_size +=
(16 - kMinRepeatedFieldAllocationSize) * prototype.SpaceUsed();
(16 - kRepeatedFieldLowerClampLimit) * prototype.SpaceUsed();
EXPECT_LE(min_expected_size, message.SpaceUsed());
}
}

@ -100,7 +100,7 @@ template <typename ImplicitWeakType>
class ImplicitWeakTypeHandler {
public:
typedef MessageLite Type;
static const bool Moveable = false;
static constexpr bool Moveable = false;
static inline MessageLite* NewFromPrototype(const MessageLite* prototype,
Arena* arena = NULL) {

@ -132,7 +132,7 @@ namespace {
class CodedStreamTest : public testing::Test {
protected:
// Buffer used during most of the tests. This assumes tests run sequentially.
static const int kBufferSize = 1024 * 64;
static constexpr int kBufferSize = 1024 * 64;
static uint8 buffer_[kBufferSize];
};

@ -153,9 +153,9 @@ template <typename Derived, typename K, typename V,
struct DeconstructMapEntry<MapEntry<Derived, K, V, key, value, default_enum> > {
typedef K Key;
typedef V Value;
static const WireFormatLite::FieldType kKeyFieldType = key;
static const WireFormatLite::FieldType kValueFieldType = value;
static const int default_enum_value = default_enum;
static constexpr WireFormatLite::FieldType kKeyFieldType = key;
static constexpr WireFormatLite::FieldType kValueFieldType = value;
static constexpr int default_enum_value = default_enum;
};
} // namespace internal

@ -196,8 +196,7 @@ bool DynamicMapField::ContainsMapKey(const MapKey& map_key) const {
}
void DynamicMapField::AllocateMapValue(MapValueRef* map_val) {
const FieldDescriptor* val_des =
default_entry_->GetDescriptor()->FindFieldByName("value");
const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value();
map_val->SetType(val_des->cpp_type());
// Allocate memory for the MapValueRef, and initialize to
// default value.
@ -300,7 +299,7 @@ void DynamicMapField::MergeFrom(const MapFieldBase& other) {
// Copy map value
const FieldDescriptor* field_descriptor =
default_entry_->GetDescriptor()->FindFieldByName("value");
default_entry_->GetDescriptor()->map_value();
switch (field_descriptor->cpp_type()) {
case FieldDescriptor::CPPTYPE_INT32: {
map_val->SetInt32Value(other_it->second.GetInt32Value());
@ -360,10 +359,8 @@ void DynamicMapField::Swap(MapFieldBase* other) {
void DynamicMapField::SyncRepeatedFieldWithMapNoLock() const {
const Reflection* reflection = default_entry_->GetReflection();
const FieldDescriptor* key_des =
default_entry_->GetDescriptor()->FindFieldByName("key");
const FieldDescriptor* val_des =
default_entry_->GetDescriptor()->FindFieldByName("value");
const FieldDescriptor* key_des = default_entry_->GetDescriptor()->map_key();
const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value();
if (MapFieldBase::repeated_field_ == NULL) {
if (MapFieldBase::arena_ == NULL) {
MapFieldBase::repeated_field_ = new RepeatedPtrField<Message>();
@ -448,10 +445,8 @@ void DynamicMapField::SyncRepeatedFieldWithMapNoLock() const {
void DynamicMapField::SyncMapWithRepeatedFieldNoLock() const {
Map<MapKey, MapValueRef>* map = &const_cast<DynamicMapField*>(this)->map_;
const Reflection* reflection = default_entry_->GetReflection();
const FieldDescriptor* key_des =
default_entry_->GetDescriptor()->FindFieldByName("key");
const FieldDescriptor* val_des =
default_entry_->GetDescriptor()->FindFieldByName("value");
const FieldDescriptor* key_des = default_entry_->GetDescriptor()->map_key();
const FieldDescriptor* val_des = default_entry_->GetDescriptor()->map_value();
// DynamicMapField owns map values. Need to delete them before clearing
// the map.
if (MapFieldBase::arena_ == nullptr) {

@ -443,7 +443,7 @@ class MapField : public TypeDefinedMapFieldBase<Key, T> {
// different exposed type in Map's api and repeated field's api. For
// details see the comment in the implementation of
// SyncMapWithRepeatedFieldNoLock.
static const bool kIsValueEnum = ValueTypeHandler::kIsEnum;
static constexpr bool kIsValueEnum = ValueTypeHandler::kIsEnum;
typedef typename MapIf<kIsValueEnum, T, const T&>::type CastValueType;
public:

@ -107,8 +107,8 @@ class MapFieldBasePrimitiveTest : public ::testing::Test {
map_descriptor_ = unittest::TestMap::descriptor()
->FindFieldByName("map_int32_int32")
->message_type();
key_descriptor_ = map_descriptor_->FindFieldByName("key");
value_descriptor_ = map_descriptor_->FindFieldByName("value");
key_descriptor_ = map_descriptor_->map_key();
value_descriptor_ = map_descriptor_->map_value();
// Build map field
map_field_.reset(new MapFieldType);

@ -238,6 +238,51 @@ inline MapReflectionTester::MapReflectionTester(
EXPECT_FALSE(map_int32_enum_val_ == nullptr);
EXPECT_FALSE(map_int32_foreign_message_key_ == nullptr);
EXPECT_FALSE(map_int32_foreign_message_val_ == nullptr);
std::vector<const FieldDescriptor*> all_map_descriptors = {
map_int32_int32_key_,
map_int32_int32_val_,
map_int64_int64_key_,
map_int64_int64_val_,
map_uint32_uint32_key_,
map_uint32_uint32_val_,
map_uint64_uint64_key_,
map_uint64_uint64_val_,
map_sint32_sint32_key_,
map_sint32_sint32_val_,
map_sint64_sint64_key_,
map_sint64_sint64_val_,
map_fixed32_fixed32_key_,
map_fixed32_fixed32_val_,
map_fixed64_fixed64_key_,
map_fixed64_fixed64_val_,
map_sfixed32_sfixed32_key_,
map_sfixed32_sfixed32_val_,
map_sfixed64_sfixed64_key_,
map_sfixed64_sfixed64_val_,
map_int32_float_key_,
map_int32_float_val_,
map_int32_double_key_,
map_int32_double_val_,
map_bool_bool_key_,
map_bool_bool_val_,
map_string_string_key_,
map_string_string_val_,
map_int32_bytes_key_,
map_int32_bytes_val_,
map_int32_enum_key_,
map_int32_enum_val_,
map_int32_foreign_message_key_,
map_int32_foreign_message_val_};
for (const FieldDescriptor* fdesc : all_map_descriptors) {
GOOGLE_CHECK(fdesc->containing_type() != nullptr) << fdesc->name();
if (fdesc->name() == "key") {
EXPECT_EQ(fdesc->containing_type()->map_key(), fdesc);
} else {
EXPECT_EQ(fdesc->name(), "value");
EXPECT_EQ(fdesc->containing_type()->map_value(), fdesc);
}
}
}
// Shorthand to get a FieldDescriptor for a field of unittest::TestMap.

@ -155,13 +155,13 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
typedef typename MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE,
Type>::TypeOnMemory TypeOnMemory;
// Corresponding wire type for field type.
static const WireFormatLite::WireType kWireType =
static constexpr WireFormatLite::WireType kWireType =
MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE, Type>::kWireType;
// Whether wire type is for message.
static const bool kIsMessage =
static constexpr bool kIsMessage =
MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE, Type>::kIsMessage;
// Whether wire type is for enum.
static const bool kIsEnum =
static constexpr bool kIsEnum =
MapWireFieldTypeTraits<WireFormatLite::TYPE_MESSAGE, Type>::kIsEnum;
// Functions used in parsing and serialization. ===================

@ -222,6 +222,9 @@ bool CreateUnknownEnumValues(const FieldDescriptor* field);
// optimized for speed will want to override these with faster implementations,
// but classes optimized for code size may be happy with keeping them. See
// the optimize_for option in descriptor.proto.
//
// Users must not derive from this class. Only the protocol compiler and
// the internal library are allowed to create subclasses.
class PROTOBUF_EXPORT Message : public MessageLite {
public:
inline Message() {}

@ -182,6 +182,9 @@ PROTOBUF_EXPORT size_t StringSpaceUsedExcludingSelfLong(const std::string& str);
// is best when you only have a small number of message types linked
// into your binary, in which case the size of the protocol buffers
// runtime itself is the biggest problem.
//
// Users must not derive from this class. Only the protocol compiler and
// the internal library are allowed to create subclasses.
class PROTOBUF_EXPORT MessageLite {
public:
inline MessageLite() {}

@ -139,8 +139,8 @@ class InternalMetadata {
// ptr_ is a Container*.
kTagContainer = 1,
};
static const intptr_t kPtrTagMask = 1;
static const intptr_t kPtrValueMask = ~kPtrTagMask;
static constexpr intptr_t kPtrTagMask = 1;
static constexpr intptr_t kPtrValueMask = ~kPtrTagMask;
// Accessors for pointer tag and pointer value.
PROTOBUF_ALWAYS_INLINE int PtrTag() const {

@ -471,7 +471,7 @@ class RepeatedFieldRefIterator
// RepeatedFieldAccessor type, etc.
template <typename T>
struct PrimitiveTraits {
static const bool is_primitive = false;
static constexpr bool is_primitive = false;
};
#define DEFINE_PRIMITIVE(TYPE, type) \
template <> \
@ -497,7 +497,8 @@ struct RefTypeTraits<
typedef T AccessorValueType;
typedef T IteratorValueType;
typedef T* IteratorPointerType;
static const FieldDescriptor::CppType cpp_type = PrimitiveTraits<T>::cpp_type;
static constexpr FieldDescriptor::CppType cpp_type =
PrimitiveTraits<T>::cpp_type;
static const Descriptor* GetMessageFieldDescriptor() { return NULL; }
};
@ -510,7 +511,7 @@ struct RefTypeTraits<
typedef int32 AccessorValueType;
typedef T IteratorValueType;
typedef int32* IteratorPointerType;
static const FieldDescriptor::CppType cpp_type =
static constexpr FieldDescriptor::CppType cpp_type =
FieldDescriptor::CPPTYPE_ENUM;
static const Descriptor* GetMessageFieldDescriptor() { return NULL; }
};
@ -523,7 +524,7 @@ struct RefTypeTraits<
typedef std::string AccessorValueType;
typedef const std::string IteratorValueType;
typedef const std::string* IteratorPointerType;
static const FieldDescriptor::CppType cpp_type =
static constexpr FieldDescriptor::CppType cpp_type =
FieldDescriptor::CPPTYPE_STRING;
static const Descriptor* GetMessageFieldDescriptor() { return NULL; }
};
@ -547,7 +548,7 @@ struct RefTypeTraits<
typedef Message AccessorValueType;
typedef const T& IteratorValueType;
typedef const T* IteratorPointerType;
static const FieldDescriptor::CppType cpp_type =
static constexpr FieldDescriptor::CppType cpp_type =
FieldDescriptor::CPPTYPE_MESSAGE;
static const Descriptor* GetMessageFieldDescriptor() {
return MessageDescriptorGetter<T>::get();

@ -56,7 +56,7 @@ void** RepeatedPtrFieldBase::InternalExtend(int extend_amount) {
}
Rep* old_rep = rep_;
Arena* arena = GetArena();
new_size = std::max(kMinRepeatedFieldAllocationSize,
new_size = std::max(internal::kRepeatedFieldLowerClampLimit,
std::max(total_size_ * 2, new_size));
GOOGLE_CHECK_LE(new_size, (std::numeric_limits<size_t>::max() - kRepHeaderSize) /
sizeof(old_rep->elements[0]))

@ -66,6 +66,7 @@
#include <type_traits>
// Must be included last.
#include <google/protobuf/port_def.inc>
#ifdef SWIG
@ -85,7 +86,16 @@ namespace internal {
class MergePartialFromCodedStreamHelper;
static const int kMinRepeatedFieldAllocationSize = 4;
// kRepeatedFieldLowerClampLimit is the smallest size that will be allocated
// when growing a repeated field.
constexpr int kRepeatedFieldLowerClampLimit = 4;
// kRepeatedFieldUpperClampLimit is the lowest signed integer value that
// overflows when multiplied by 2 (which is undefined behavior). Sizes above
// this will clamp to the maximum int value instead of following exponential
// growth when growing a repeated field.
constexpr int kRepeatedFieldUpperClampLimit =
(std::numeric_limits<int>::max() / 2) + 1;
// A utility function for logging that doesn't need any template types.
void LogIndexOutOfBounds(int index, int size);
@ -309,7 +319,7 @@ class RepeatedField final {
inline void InternalSwap(RepeatedField* other);
private:
static const int kInitialSize = 0;
static constexpr int kInitialSize = 0;
// A note on the representation here (see also comment below for
// RepeatedPtrFieldBase's struct Rep):
//
@ -390,6 +400,84 @@ class RepeatedField final {
}
}
}
// This class is a performance wrapper around RepeatedField::Add(const T&)
// function. In general unless a RepeatedField is a local stack variable LLVM
// has a hard time optimizing Add. The machine code tends to be
// loop:
// mov %size, dword ptr [%repeated_field] // load
// cmp %size, dword ptr [%repeated_field + 4]
// jae fallback
// mov %buffer, qword ptr [%repeated_field + 8]
// mov dword [%buffer + %size * 4], %value
// inc %size // increment
// mov dword ptr [%repeated_field], %size // store
// jmp loop
//
// This puts a load/store in each iteration of the important loop variable
// size. It's a pretty bad compile that happens even in simple cases, but
// largely the presence of the fallback path disturbs the compilers mem-to-reg
// analysis.
//
// This class takes ownership of a repeated field for the duration of it's
// lifetime. The repeated field should not be accessed during this time, ie.
// only access through this class is allowed. This class should always be a
// function local stack variable. Intended use
//
// void AddSequence(const int* begin, const int* end, RepeatedField<int>* out)
// {
// RepeatedFieldAdder<int> adder(out); // Take ownership of out
// for (auto it = begin; it != end; ++it) {
// adder.Add(*it);
// }
// }
//
// Typically due to the fact adder is a local stack variable. The compiler
// will be successful in mem-to-reg transformation and the machine code will
// be loop: cmp %size, %capacity jae fallback mov dword ptr [%buffer + %size *
// 4], %val inc %size jmp loop
//
// The first version executes at 7 cycles per iteration while the second
// version near 1 or 2 cycles.
class FastAdder {
public:
explicit FastAdder(RepeatedField* rf) : repeated_field_(rf) {
if (kIsPod) {
index_ = repeated_field_->current_size_;
capacity_ = repeated_field_->total_size_;
buffer_ = repeated_field_->unsafe_elements();
}
}
~FastAdder() {
if (kIsPod) repeated_field_->current_size_ = index_;
}
void Add(const Element& val) {
if (kIsPod) {
if (index_ == capacity_) {
repeated_field_->current_size_ = index_;
repeated_field_->Reserve(index_ + 1);
capacity_ = repeated_field_->total_size_;
buffer_ = repeated_field_->unsafe_elements();
}
buffer_[index_++] = val;
} else {
repeated_field_->Add(val);
}
}
private:
constexpr static bool kIsPod = std::is_pod<Element>::value;
RepeatedField* repeated_field_;
int index_;
int capacity_;
Element* buffer_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FastAdder);
};
friend class TestRepeatedFieldHelper;
friend class ::google::protobuf::internal::ParseContext;
};
template <typename Element>
@ -629,7 +717,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
inline Arena* GetArena() const { return arena_; }
private:
static const int kInitialSize = 0;
static constexpr int kInitialSize = 0;
// A few notes on internal representation:
//
// We use an indirected approach, with struct Rep, to keep
@ -648,7 +736,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
int allocated_size;
void* elements[1];
};
static const size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*);
static constexpr size_t kRepHeaderSize = sizeof(Rep) - sizeof(void*);
Rep* rep_;
template <typename TypeHandler>
@ -1240,14 +1328,19 @@ inline void RepeatedField<Element>::Set(int index, const Element& value) {
template <typename Element>
inline void RepeatedField<Element>::Add(const Element& value) {
if (current_size_ == total_size_) Reserve(total_size_ + 1);
elements()[current_size_++] = value;
uint32 size = current_size_;
if (static_cast<int>(size) == total_size_) Reserve(total_size_ + 1);
elements()[size] = value;
current_size_ = size + 1;
}
template <typename Element>
inline Element* RepeatedField<Element>::Add() {
if (current_size_ == total_size_) Reserve(total_size_ + 1);
return &elements()[current_size_++];
uint32 size = current_size_;
if (static_cast<int>(size) == total_size_) Reserve(total_size_ + 1);
auto ptr = &elements()[size];
current_size_ = size + 1;
return ptr;
}
template <typename Element>
@ -1269,9 +1362,8 @@ inline void RepeatedField<Element>::Add(Iter begin, Iter end) {
std::copy(begin, end, elements() + size());
current_size_ = reserve + size();
} else {
for (; begin != end; ++begin) {
Add(*begin);
}
FastAdder fast_adder(this);
for (; begin != end; ++begin) fast_adder.Add(*begin);
}
}
@ -1425,6 +1517,30 @@ inline size_t RepeatedField<Element>::SpaceUsedExcludingSelfLong() const {
return total_size_ > 0 ? (total_size_ * sizeof(Element) + kRepHeaderSize) : 0;
}
namespace internal {
// Returns the new size for a reserved field based on its 'total_size' and the
// requested 'new_size'. The result is clamped to the closed interval:
// [internal::kMinRepeatedFieldAllocationSize,
// std::numeric_limits<int>::max()]
// Requires:
// new_size > total_size &&
// (total_size == 0 ||
// total_size >= kRepeatedFieldLowerClampLimit)
inline int CalculateReserveSize(int total_size, int new_size) {
if (new_size < kRepeatedFieldLowerClampLimit) {
// Clamp to smallest allowed size.
return kRepeatedFieldLowerClampLimit;
}
if (total_size < kRepeatedFieldUpperClampLimit) {
return std::max(total_size * 2, new_size);
} else {
// Clamp to largest allowed size.
GOOGLE_DCHECK_GT(new_size, kRepeatedFieldUpperClampLimit);
return std::numeric_limits<int>::max();
}
}
} // namespace internal
// Avoid inlining of Reserve(): new, copy, and delete[] lead to a significant
// amount of code bloat.
template <typename Element>
@ -1433,8 +1549,7 @@ void RepeatedField<Element>::Reserve(int new_size) {
Rep* old_rep = total_size_ > 0 ? rep() : NULL;
Rep* new_rep;
Arena* arena = GetArena();
new_size = std::max(internal::kMinRepeatedFieldAllocationSize,
std::max(total_size_ * 2, new_size));
new_size = internal::CalculateReserveSize(total_size_, new_size);
GOOGLE_DCHECK_LE(
static_cast<size_t>(new_size),
(std::numeric_limits<size_t>::max() - kRepHeaderSize) / sizeof(Element))
@ -1448,6 +1563,10 @@ void RepeatedField<Element>::Reserve(int new_size) {
}
new_rep->arena = arena;
int old_total_size = total_size_;
// Already known: new_size >= internal::kMinRepeatedFieldAllocationSize
// Maintain invariant:
// total_size_ == 0 ||
// total_size_ >= internal::kMinRepeatedFieldAllocationSize
total_size_ = new_size;
arena_or_elements_ = new_rep->elements;
// Invoke placement-new on newly allocated elements. We shouldn't have to do

@ -38,6 +38,8 @@
#include <google/protobuf/repeated_field.h>
#include <algorithm>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <list>
#include <sstream>
@ -53,6 +55,9 @@
#include <gtest/gtest.h>
#include <google/protobuf/stubs/stl_util.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
namespace {
@ -268,6 +273,68 @@ TEST(RepeatedField, Resize) {
EXPECT_TRUE(field.empty());
}
TEST(RepeatedField, ReserveNothing) {
RepeatedField<int> field;
EXPECT_EQ(0, field.Capacity());
field.Reserve(-1);
EXPECT_EQ(0, field.Capacity());
}
TEST(RepeatedField, ReserveLowerClamp) {
const int clamped_value = internal::CalculateReserveSize(0, 1);
EXPECT_EQ(internal::kRepeatedFieldLowerClampLimit, clamped_value);
EXPECT_EQ(clamped_value, internal::CalculateReserveSize(clamped_value, 2));
}
TEST(RepeatedField, ReserveGrowth) {
// Make sure the field capacity doubles in size on repeated reservation.
for (int size = internal::kRepeatedFieldLowerClampLimit, i = 0; i < 4;
++i, size *= 2) {
EXPECT_EQ(size * 2, internal::CalculateReserveSize(size, size + 1));
}
}
TEST(RepeatedField, ReserveLarge) {
const int old_size = 10;
// This is a size we won't get by doubling:
const int new_size = old_size * 3 + 1;
// Reserving more than 2x current capacity should grow directly to that size.
EXPECT_EQ(new_size, internal::CalculateReserveSize(old_size, new_size));
}
TEST(RepeatedField, ReserveHuge) {
// Largest value that does not clamp to the large limit:
constexpr int non_clamping_limit = std::numeric_limits<int>::max() / 2;
ASSERT_LT(2 * non_clamping_limit, std::numeric_limits<int>::max());
EXPECT_LT(internal::CalculateReserveSize(non_clamping_limit,
non_clamping_limit + 1),
std::numeric_limits<int>::max());
// Smallest size that *will* clamp to the upper limit:
constexpr int min_clamping_size = std::numeric_limits<int>::max() / 2 + 1;
EXPECT_EQ(
internal::CalculateReserveSize(min_clamping_size, min_clamping_size + 1),
std::numeric_limits<int>::max());
#ifdef PROTOBUF_TEST_ALLOW_LARGE_ALLOC
// The rest of this test may allocate several GB of memory, so it is only
// built if explicitly requested.
RepeatedField<int> huge_field;
// Reserve a size for huge_field that will clamp.
huge_field.Reserve(min_clamping_size);
EXPECT_GE(huge_field.Capacity(), min_clamping_size);
ASSERT_LT(huge_field.Capacity(), std::numeric_limits<int>::max() - 1);
// Allocation may return more memory than we requested. However, the updated
// size must still be clamped to a valid range.
huge_field.Reserve(huge_field.Capacity() + 1);
EXPECT_EQ(huge_field.Capacity(), std::numeric_limits<int>::max());
#endif // PROTOBUF_TEST_ALLOW_LARGE_ALLOC
}
TEST(RepeatedField, MergeFrom) {
RepeatedField<int> source, destination;
source.Add(4);

@ -384,9 +384,8 @@ bool DataPiece::DecodeBase64(StringPiece src, std::string* dest) const {
if (Base64Unescape(src, dest)) {
if (use_strict_base64_decoding_) {
std::string encoded;
Base64Escape(
reinterpret_cast<const unsigned char*>(dest->data()), dest->length(),
&encoded, false);
Base64Escape(reinterpret_cast<const unsigned char*>(dest->data()),
dest->length(), &encoded, false);
StringPiece src_no_padding = StringPiece(src).substr(
0, HasSuffixString(src, "=") ? src.find_last_not_of('=') + 1
: src.length());

@ -44,34 +44,34 @@ class JsonEscaping {
// The minimum value of a unicode high-surrogate code unit in the utf-16
// encoding. A high-surrogate is also known as a leading-surrogate.
// See http://www.unicode.org/glossary/#high_surrogate_code_unit
static const uint16 kMinHighSurrogate = 0xd800;
static constexpr uint16 kMinHighSurrogate = 0xd800;
// The maximum value of a unicide high-surrogate code unit in the utf-16
// encoding. A high-surrogate is also known as a leading-surrogate.
// See http://www.unicode.org/glossary/#high_surrogate_code_unit
static const uint16 kMaxHighSurrogate = 0xdbff;
static constexpr uint16 kMaxHighSurrogate = 0xdbff;
// The minimum value of a unicode low-surrogate code unit in the utf-16
// encoding. A low-surrogate is also known as a trailing-surrogate.
// See http://www.unicode.org/glossary/#low_surrogate_code_unit
static const uint16 kMinLowSurrogate = 0xdc00;
static constexpr uint16 kMinLowSurrogate = 0xdc00;
// The maximum value of a unicode low-surrogate code unit in the utf-16
// encoding. A low-surrogate is also known as a trailing surrogate.
// See http://www.unicode.org/glossary/#low_surrogate_code_unit
static const uint16 kMaxLowSurrogate = 0xdfff;
static constexpr uint16 kMaxLowSurrogate = 0xdfff;
// The minimum value of a unicode supplementary code point.
// See http://www.unicode.org/glossary/#supplementary_code_point
static const uint32 kMinSupplementaryCodePoint = 0x010000;
static constexpr uint32 kMinSupplementaryCodePoint = 0x010000;
// The minimum value of a unicode code point.
// See http://www.unicode.org/glossary/#code_point
static const uint32 kMinCodePoint = 0x000000;
static constexpr uint32 kMinCodePoint = 0x000000;
// The maximum value of a unicode code point.
// See http://www.unicode.org/glossary/#code_point
static const uint32 kMaxCodePoint = 0x10ffff;
static constexpr uint32 kMaxCodePoint = 0x10ffff;
JsonEscaping() {}
virtual ~JsonEscaping() {}

@ -537,6 +537,11 @@ Status ProtoStreamObjectSource::RenderStruct(const ProtoStreamObjectSource* os,
ow->StartObject(field_name);
while (tag != 0) {
field = os->FindAndVerifyField(type, tag);
if (field == nullptr) {
WireFormat::SkipField(os->stream_, tag, nullptr);
tag = os->stream_->ReadTag();
continue;
}
// google.protobuf.Struct has only one field that is a map. Hence we use
// RenderMap to render that field.
if (os->IsMap(*field)) {

@ -119,10 +119,10 @@ TEST(TimeUtilTest, DurationStringFormat) {
// Duration must support range from -315,576,000,000s to +315576000000s
// which includes negative values.
EXPECT_TRUE(TimeUtil::FromString("315576000000.999999999s", &d));
EXPECT_EQ(315576000000LL, d.seconds());
EXPECT_EQ(int64{315576000000}, d.seconds());
EXPECT_EQ(999999999, d.nanos());
EXPECT_TRUE(TimeUtil::FromString("-315576000000.999999999s", &d));
EXPECT_EQ(-315576000000LL, d.seconds());
EXPECT_EQ(int64{-315576000000}, d.seconds());
EXPECT_EQ(-999999999, d.nanos());
}
@ -278,20 +278,22 @@ TEST(TimeUtilTest, DurationOperators) {
// Multiplication should not overflow if the result fits into the supported
// range of Duration (intermediate result may be larger than int64).
EXPECT_EQ("315575999684.424s",
TimeUtil::ToString((one_second - one_nano) * 315576000000LL));
TimeUtil::ToString((one_second - one_nano) * int64{315576000000}));
EXPECT_EQ("-315575999684.424s",
TimeUtil::ToString((one_nano - one_second) * 315576000000LL));
EXPECT_EQ("-315575999684.424s",
TimeUtil::ToString((one_second - one_nano) * (-315576000000LL)));
TimeUtil::ToString((one_nano - one_second) * int64{315576000000}));
EXPECT_EQ("-315575999684.424s", TimeUtil::ToString((one_second - one_nano) *
(int64{-315576000000})));
// Test / and %
EXPECT_EQ("0.999999999s", TimeUtil::ToString(a / 2));
EXPECT_EQ("-0.999999999s", TimeUtil::ToString(b / 2));
Duration large = TimeUtil::SecondsToDuration(315576000000LL) - one_nano;
Duration large = TimeUtil::SecondsToDuration(int64{315576000000}) - one_nano;
// We have to handle division with values beyond 64 bits.
EXPECT_EQ("0.999999999s", TimeUtil::ToString(large / 315576000000LL));
EXPECT_EQ("-0.999999999s", TimeUtil::ToString((-large) / 315576000000LL));
EXPECT_EQ("-0.999999999s", TimeUtil::ToString(large / (-315576000000LL)));
EXPECT_EQ("0.999999999s", TimeUtil::ToString(large / int64{315576000000}));
EXPECT_EQ("-0.999999999s",
TimeUtil::ToString((-large) / int64{315576000000}));
EXPECT_EQ("-0.999999999s",
TimeUtil::ToString(large / (int64{-315576000000})));
Duration large2 = large + one_nano;
EXPECT_EQ(large, large % large2);
EXPECT_EQ(-large, (-large) % large2);

@ -156,9 +156,9 @@ class PROTOBUF_EXPORT WireFormatLite {
}
// Number of bits in a tag which identify the wire type.
static const int kTagTypeBits = 3;
static constexpr int kTagTypeBits = 3;
// Mask for those bits.
static const uint32 kTagTypeMask = (1 << kTagTypeBits) - 1;
static constexpr uint32 kTagTypeMask = (1 << kTagTypeBits) - 1;
// Helper functions for encoding and decoding tags. (Inlined below and in
// _inl.h)
@ -210,9 +210,9 @@ class PROTOBUF_EXPORT WireFormatLite {
// required string message = 3;
// }
// }
static const int kMessageSetItemNumber = 1;
static const int kMessageSetTypeIdNumber = 2;
static const int kMessageSetMessageNumber = 3;
static constexpr int kMessageSetItemNumber = 1;
static constexpr int kMessageSetTypeIdNumber = 2;
static constexpr int kMessageSetMessageNumber = 3;
static const int kMessageSetItemStartTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(
kMessageSetItemNumber, WireFormatLite::WIRETYPE_START_GROUP);
static const int kMessageSetItemEndTag = GOOGLE_PROTOBUF_WIRE_FORMAT_MAKE_TAG(
@ -685,13 +685,13 @@ class PROTOBUF_EXPORT WireFormatLite {
static size_t EnumSize(const RepeatedField<int>& value);
// These types always have the same size.
static const size_t kFixed32Size = 4;
static const size_t kFixed64Size = 8;
static const size_t kSFixed32Size = 4;
static const size_t kSFixed64Size = 8;
static const size_t kFloatSize = 4;
static const size_t kDoubleSize = 8;
static const size_t kBoolSize = 1;
static constexpr size_t kFixed32Size = 4;
static constexpr size_t kFixed64Size = 8;
static constexpr size_t kSFixed32Size = 4;
static constexpr size_t kSFixed64Size = 8;
static constexpr size_t kFloatSize = 4;
static constexpr size_t kDoubleSize = 8;
static constexpr size_t kBoolSize = 1;
static inline size_t StringSize(const std::string& value);
static inline size_t BytesSize(const std::string& value);

Loading…
Cancel
Save