From 66540237ca212401ec1e279224b8db40f52e4ab9 Mon Sep 17 00:00:00 2001 From: Hao Nguyen Date: Fri, 17 May 2019 12:01:17 -0700 Subject: [PATCH] Down integrate to GitHub --- benchmarks/python/py_benchmark.py | 4 +- .../google/protobuf/multiple_files_test.proto | 2 +- .../protobuf/test_bad_identifiers.proto | 2 +- protobuf.pc.in | 1 + python/google/protobuf/descriptor_pool.py | 24 ++++---- .../internal/descriptor_database_test.py | 5 +- .../protobuf/internal/descriptor_pool_test.py | 41 +++++-------- .../protobuf/internal/python_message.py | 6 +- .../protobuf/compiler/js/js_generator.cc | 14 +++-- src/google/protobuf/descriptor.cc | 1 + src/google/protobuf/descriptor_database.cc | 18 +++--- src/google/protobuf/descriptor_database.h | 6 +- .../protobuf/generated_message_reflection.cc | 59 +++++++++---------- src/google/protobuf/util/time_util.h | 6 +- 14 files changed, 95 insertions(+), 94 deletions(-) diff --git a/benchmarks/python/py_benchmark.py b/benchmarks/python/py_benchmark.py index f2c7bccbe8..60e377e4b8 100755 --- a/benchmarks/python/py_benchmark.py +++ b/benchmarks/python/py_benchmark.py @@ -41,7 +41,7 @@ import benchmarks_pb2 as benchmarks_pb2 def run_one_test(filename): - data = open(filename).read() + data = open(filename, "rb").read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) total_bytes = 0 @@ -69,7 +69,7 @@ def init(filename): message_list=[] counter = 0 total_bytes = 0 - data = open(filename).read() + data = open(filename, "rb").read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) diff --git a/java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto b/java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto index fb3a74b978..0de6f7b868 100644 --- a/java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto @@ -41,7 +41,7 @@ package protobuf_unittest; import "google/protobuf/unittest.proto"; import "google/protobuf/descriptor.proto"; -option java_generic_services = true; // auto-added +option java_generic_services = true; // auto-added option java_multiple_files = true; option java_outer_classname = "MultipleFilesTestProto"; diff --git a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto index df43e9b922..8bf1691431 100644 --- a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto +++ b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto @@ -39,7 +39,7 @@ syntax = "proto2"; // See: http://go/proto2-generic-services-default package io_protocol_tests; -option java_generic_services = true; // auto-added +option java_generic_services = true; // auto-added option java_package = "com.google.protobuf"; option java_outer_classname = "TestBadIdentifiersProto"; diff --git a/protobuf.pc.in b/protobuf.pc.in index 4e88b514a2..055a9d0563 100644 --- a/protobuf.pc.in +++ b/protobuf.pc.in @@ -8,5 +8,6 @@ Description: Google's Data Interchange Format Version: @VERSION@ Libs: -L${libdir} -lprotobuf @PTHREAD_LIBS@ Libs.private: @LIBS@ + Cflags: -I${includedir} @PTHREAD_CFLAGS@ Conflicts: protobuf-lite diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index 7309ac2a77..bc5029a557 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -99,6 +99,8 @@ def _IsMessageSetExtension(field): field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL) + + class DescriptorPool(object): """A collection of protobufs dynamically constructed by descriptor protos.""" @@ -161,18 +163,18 @@ class DescriptorPool(object): if not isinstance(desc, descriptor_type) or ( old_file != file_name): - warn_msg = ('Conflict register for file "' + file_name + - '": ' + desc_name + - ' is already defined in file "' + - old_file + '". Please fix the conflict by adding ' - 'package name on the proto file, or use different ' - 'name for the duplication. This warning will ' - 'turn into error soon.') + error_msg = ('Conflict register for file "' + file_name + + '": ' + desc_name + + ' is already defined in file "' + + old_file + '". Please fix the conflict by adding ' + 'package name on the proto file, or use different ' + 'name for the duplication.') if isinstance(desc, descriptor.EnumValueDescriptor): - warn_msg += ('\nNote: enum values appear as ' - 'siblings of the enum type instead of ' - 'children of it.') - warnings.warn(warn_msg, RuntimeWarning) + error_msg += ('\nNote: enum values appear as ' + 'siblings of the enum type instead of ' + 'children of it.') + + raise TypeError(error_msg) return diff --git a/python/google/protobuf/internal/descriptor_database_test.py b/python/google/protobuf/internal/descriptor_database_test.py index a185612937..619779a5dd 100644 --- a/python/google/protobuf/internal/descriptor_database_test.py +++ b/python/google/protobuf/internal/descriptor_database_test.py @@ -115,19 +115,18 @@ class DescriptorDatabaseTest(unittest.TestCase): db.Add(unittest_fd) conflict_fd = descriptor_pb2.FileDescriptorProto.FromString( unittest_pb2.DESCRIPTOR.serialized_pb) - conflict_fd.name = 'other_file' + conflict_fd.name = 'other_file2' with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter('always') db.Add(conflict_fd) self.assertTrue(len(w)) self.assertIs(w[0].category, RuntimeWarning) - self.assertIn('Conflict register for file "other_file": ', + self.assertIn('Conflict register for file "other_file2": ', str(w[0].message)) self.assertIn('already defined in file ' '"google/protobuf/unittest.proto"', str(w[0].message)) - if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index bf7e346a90..9414f6f5af 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -36,7 +36,6 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' import copy import os -import warnings try: import unittest2 as unittest #PY26 @@ -532,33 +531,21 @@ class DescriptorPoolTestBase(object): if api_implementation.Type() == 'cpp': pass else: - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter('always') - pool = copy.deepcopy(self.pool) - # No warnings to add the same descriptors. - file_descriptor = unittest_pb2.DESCRIPTOR - pool.AddDescriptor( - file_descriptor.message_types_by_name['TestAllTypes']) - pool.AddEnumDescriptor( - file_descriptor.enum_types_by_name['ForeignEnum']) - pool.AddServiceDescriptor( - file_descriptor.services_by_name['TestService']) - pool.AddExtensionDescriptor( - file_descriptor.extensions_by_name['optional_int32_extension']) - self.assertEqual(len(w), 0) - # Check warnings for conflict descriptors with the same name. - pool.Add(unittest_fd) - pool.Add(conflict_fd) - pool.FindFileByName(unittest_fd.name) + pool = copy.deepcopy(self.pool) + file_descriptor = unittest_pb2.DESCRIPTOR + pool.AddDescriptor( + file_descriptor.message_types_by_name['TestAllTypes']) + pool.AddEnumDescriptor( + file_descriptor.enum_types_by_name['ForeignEnum']) + pool.AddServiceDescriptor( + file_descriptor.services_by_name['TestService']) + pool.AddExtensionDescriptor( + file_descriptor.extensions_by_name['optional_int32_extension']) + pool.Add(unittest_fd) + pool.Add(conflict_fd) + pool.FindFileByName(unittest_fd.name) + with self.assertRaises(TypeError): pool.FindFileByName(conflict_fd.name) - self.assertTrue(len(w)) - self.assertIs(w[0].category, RuntimeWarning) - self.assertIn('Conflict register for file "other_file": ', - str(w[0].message)) - self.assertIn('already defined in file ' - '"google/protobuf/unittest.proto"', - str(w[0].message)) @testing_refleaks.TestCase diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 14cb749277..803ae84849 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -692,7 +692,11 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls): # pylint: disable=protected-access # Testing the value for truthiness captures all of the proto3 defaults # (0, 0.0, enum 0, and False). - new_value = type_checker.CheckValue(new_value) + try: + new_value = type_checker.CheckValue(new_value) + except TypeError as e: + raise TypeError( + 'Cannot set %s to %.1024r: %s' % (field.full_name, new_value, e)) if clear_when_set_to_default and not new_value: self._fields.pop(field, None) else: diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index 639759923c..0d7847f1eb 100644 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -2426,18 +2426,22 @@ void Generator::GenerateObjectTypedef(const GeneratorOptions& options, "method.\n" " * @record\n" " */\n" - "$typeName$ = function() {};\n\n", + "$typeName$ = function() {\n", "messageName", desc->name(), "typeName", type_name); for (int i = 0; i < desc->field_count(); i++) { + if (i > 0) { + printer->Print("\n"); + } printer->Print( - "/** @type {$fieldType$|undefined} */\n" - "$typeName$.prototype.$fieldName$;\n\n", - "typeName", type_name, "fieldName", - JSObjectFieldName(options, desc->field(i)), + " /** @type {$fieldType$|undefined} */\n" + " this.$fieldName$;\n", + "fieldName", JSObjectFieldName(options, desc->field(i)), // TODO(b/121097361): Add type checking for field values. "fieldType", "?"); } + + printer->Print("};\n\n"); } void Generator::GenerateClassFromObject(const GeneratorOptions& options, diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 2503923e89..1bae6a4322 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -1487,6 +1487,7 @@ const MethodDescriptor* DescriptorPool::FindMethodByName( const FieldDescriptor* DescriptorPool::FindExtensionByNumber( const Descriptor* extendee, int number) const { + if (extendee->extension_range_count() == 0) return nullptr; // A faster path to reduce lock contention in finding extensions, assuming // most extensions will be cache hit. if (mutex_ != nullptr) { diff --git a/src/google/protobuf/descriptor_database.cc b/src/google/protobuf/descriptor_database.cc index c9d35047ef..975930076e 100644 --- a/src/google/protobuf/descriptor_database.cc +++ b/src/google/protobuf/descriptor_database.cc @@ -127,14 +127,15 @@ bool SimpleDescriptorDatabase::DescriptorIndex::AddFile( for (int i = 0; i < file.message_type_size(); i++) { if (!AddSymbol(path + file.message_type(i).name(), value)) return false; - if (!AddNestedExtensions(file.message_type(i), value)) return false; + if (!AddNestedExtensions(file.name(), file.message_type(i), value)) + return false; } for (int i = 0; i < file.enum_type_size(); i++) { if (!AddSymbol(path + file.enum_type(i).name(), value)) return false; } for (int i = 0; i < file.extension_size(); i++) { if (!AddSymbol(path + file.extension(i).name(), value)) return false; - if (!AddExtension(file.extension(i), value)) return false; + if (!AddExtension(file.name(), file.extension(i), value)) return false; } for (int i = 0; i < file.service_size(); i++) { if (!AddSymbol(path + file.service(i).name(), value)) return false; @@ -203,19 +204,22 @@ bool SimpleDescriptorDatabase::DescriptorIndex::AddSymbol( template bool SimpleDescriptorDatabase::DescriptorIndex::AddNestedExtensions( - const DescriptorProto& message_type, Value value) { + const std::string& filename, const DescriptorProto& message_type, + Value value) { for (int i = 0; i < message_type.nested_type_size(); i++) { - if (!AddNestedExtensions(message_type.nested_type(i), value)) return false; + if (!AddNestedExtensions(filename, message_type.nested_type(i), value)) + return false; } for (int i = 0; i < message_type.extension_size(); i++) { - if (!AddExtension(message_type.extension(i), value)) return false; + if (!AddExtension(filename, message_type.extension(i), value)) return false; } return true; } template bool SimpleDescriptorDatabase::DescriptorIndex::AddExtension( - const FieldDescriptorProto& field, Value value) { + const std::string& filename, const FieldDescriptorProto& field, + Value value) { if (!field.extendee().empty() && field.extendee()[0] == '.') { // The extension is fully-qualified. We can use it as a lookup key in // the by_symbol_ table. @@ -226,7 +230,7 @@ bool SimpleDescriptorDatabase::DescriptorIndex::AddExtension( GOOGLE_LOG(ERROR) << "Extension conflicts with extension already in database: " "extend " << field.extendee() << " { " << field.name() << " = " - << field.number() << " }"; + << field.number() << " } from:" << filename; return false; } } else { diff --git a/src/google/protobuf/descriptor_database.h b/src/google/protobuf/descriptor_database.h index 07a5aff35f..351e6ee72b 100644 --- a/src/google/protobuf/descriptor_database.h +++ b/src/google/protobuf/descriptor_database.h @@ -199,8 +199,10 @@ class PROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase { // to the index. bool AddFile(const FileDescriptorProto& file, Value value); bool AddSymbol(const std::string& name, Value value); - bool AddNestedExtensions(const DescriptorProto& message_type, Value value); - bool AddExtension(const FieldDescriptorProto& field, Value value); + bool AddNestedExtensions(const std::string& filename, + const DescriptorProto& message_type, Value value); + bool AddExtension(const std::string& filename, + const FieldDescriptorProto& field, Value value); Value FindFile(const std::string& filename); Value FindSymbol(const std::string& name); diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 34f0f8eee2..e5718a3d91 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -1015,8 +1015,7 @@ struct FieldNumberSorter { } }; -bool IsIndexInHasBitSet(const uint32* has_bit_set, - uint32 has_bit_index) { +bool IsIndexInHasBitSet(const uint32* has_bit_set, uint32 has_bit_index) { GOOGLE_DCHECK_NE(has_bit_index, ~0u); return ((has_bit_set[has_bit_index / 32] >> (has_bit_index % 32)) & static_cast(1)) != 0; @@ -1874,8 +1873,8 @@ uint32* Reflection::MutableHasBits(Message* message) const { return GetPointerAtOffset(message, schema_.HasBitsOffset()); } -uint32 Reflection::GetOneofCase( - const Message& message, const OneofDescriptor* oneof_descriptor) const { +uint32 Reflection::GetOneofCase(const Message& message, + const OneofDescriptor* oneof_descriptor) const { return GetConstRefAtOffset( message, schema_.GetOneofCaseOffset(oneof_descriptor)); } @@ -1886,8 +1885,7 @@ uint32* Reflection::MutableOneofCase( message, schema_.GetOneofCaseOffset(oneof_descriptor)); } -const ExtensionSet& Reflection::GetExtensionSet( - const Message& message) const { +const ExtensionSet& Reflection::GetExtensionSet(const Message& message) const { return GetConstRefAtOffset(message, schema_.GetExtensionSetOffset()); } @@ -1901,8 +1899,8 @@ Arena* Reflection::GetArena(Message* message) const { return GetInternalMetadataWithArena(*message).arena(); } -const InternalMetadataWithArena& -Reflection::GetInternalMetadataWithArena(const Message& message) const { +const InternalMetadataWithArena& Reflection::GetInternalMetadataWithArena( + const Message& message) const { return GetConstRefAtOffset( message, schema_.GetMetadataOffset()); } @@ -1920,7 +1918,7 @@ const Type& Reflection::DefaultRaw(const FieldDescriptor* field) const { // Simple accessors for manipulating has_bits_. bool Reflection::HasBit(const Message& message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { GOOGLE_DCHECK(!field->options().weak()); if (schema_.HasHasbits()) { return IsIndexInHasBitSet(GetHasBits(message), schema_.HasBitIndex(field)); @@ -1980,8 +1978,7 @@ bool Reflection::HasBit(const Message& message, } } -void Reflection::SetBit(Message* message, - const FieldDescriptor* field) const { +void Reflection::SetBit(Message* message, const FieldDescriptor* field) const { GOOGLE_DCHECK(!field->options().weak()); if (!schema_.HasHasbits()) { return; @@ -1992,7 +1989,7 @@ void Reflection::SetBit(Message* message, } void Reflection::ClearBit(Message* message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { GOOGLE_DCHECK(!field->options().weak()); if (!schema_.HasHasbits()) { return; @@ -2003,7 +2000,7 @@ void Reflection::ClearBit(Message* message, } void Reflection::SwapBit(Message* message1, Message* message2, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { GOOGLE_DCHECK(!field->options().weak()); if (!schema_.HasHasbits()) { return; @@ -2027,24 +2024,24 @@ bool Reflection::HasOneof(const Message& message, } bool Reflection::HasOneofField(const Message& message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { return (GetOneofCase(message, field->containing_oneof()) == field->number()); } void Reflection::SetOneofCase(Message* message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { *MutableOneofCase(message, field->containing_oneof()) = field->number(); } void Reflection::ClearOneofField(Message* message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { if (HasOneofField(*message, field)) { ClearOneof(message, field->containing_oneof()); } } -void Reflection::ClearOneof( - Message* message, const OneofDescriptor* oneof_descriptor) const { +void Reflection::ClearOneof(Message* message, + const OneofDescriptor* oneof_descriptor) const { // TODO(jieluo): Consider to cache the unused object instead of deleting // it. It will be much faster if an application switches a lot from // a few oneof fields. Time/space tradeoff @@ -2118,13 +2115,13 @@ void* Reflection::MutableRawRepeatedString(Message* message, // used for all types except messages. template const Type& Reflection::GetField(const Message& message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { return GetRaw(message, field); } template void Reflection::SetField(Message* message, const FieldDescriptor* field, - const Type& value) const { + const Type& value) const { if (field->containing_oneof() && !HasOneofField(*message, field)) { ClearOneof(message, field->containing_oneof()); } @@ -2135,7 +2132,7 @@ void Reflection::SetField(Message* message, const FieldDescriptor* field, template Type* Reflection::MutableField(Message* message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { field->containing_oneof() ? SetOneofCase(message, field) : SetBit(message, field); return MutableRaw(message, field); @@ -2143,29 +2140,29 @@ Type* Reflection::MutableField(Message* message, template const Type& Reflection::GetRepeatedField(const Message& message, - const FieldDescriptor* field, - int index) const { + const FieldDescriptor* field, + int index) const { return GetRaw >(message, field).Get(index); } template const Type& Reflection::GetRepeatedPtrField(const Message& message, - const FieldDescriptor* field, - int index) const { + const FieldDescriptor* field, + int index) const { return GetRaw >(message, field).Get(index); } template void Reflection::SetRepeatedField(Message* message, - const FieldDescriptor* field, - int index, Type value) const { + const FieldDescriptor* field, int index, + Type value) const { MutableRaw >(message, field)->Set(index, value); } template Type* Reflection::MutableRepeatedField(Message* message, - const FieldDescriptor* field, - int index) const { + const FieldDescriptor* field, + int index) const { RepeatedPtrField* repeated = MutableRaw >(message, field); return repeated->Mutable(index); @@ -2173,13 +2170,13 @@ Type* Reflection::MutableRepeatedField(Message* message, template void Reflection::AddField(Message* message, const FieldDescriptor* field, - const Type& value) const { + const Type& value) const { MutableRaw >(message, field)->Add(value); } template Type* Reflection::AddField(Message* message, - const FieldDescriptor* field) const { + const FieldDescriptor* field) const { RepeatedPtrField* repeated = MutableRaw >(message, field); return repeated->Add(); diff --git a/src/google/protobuf/util/time_util.h b/src/google/protobuf/util/time_util.h index 8906beb2b2..9ab94cbd50 100644 --- a/src/google/protobuf/util/time_util.h +++ b/src/google/protobuf/util/time_util.h @@ -39,12 +39,12 @@ #ifdef _MSC_VER #ifdef _XBOX_ONE struct timeval { - long tv_sec; /* seconds */ - long tv_usec; /* and microseconds */ + int64 tv_sec; /* seconds */ + int64 tv_usec; /* and microseconds */ }; #else #include -#endif // _XBOX_ONE +#endif // _XBOX_ONE #else #include #endif