diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 3cf98162fe..0e03ebd14e 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -1143,7 +1143,7 @@ public abstract class ByteString implements Iterable, Serializable { synchronized (this) { // Copy the information we need into local variables so as to hold // the lock for as short a time as possible. - cachedFlushBuffers = flushedBuffers.toArray(new ByteString[flushedBuffers.size()]); + cachedFlushBuffers = flushedBuffers.toArray(new ByteString[0]); cachedBuffer = buffer; cachedBufferPos = bufferPos; } diff --git a/src/Makefile.am b/src/Makefile.am index 42975d11e2..e10ed063d5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -177,6 +177,7 @@ nobase_include_HEADERS = \ google/protobuf/util/field_comparator.h \ google/protobuf/util/field_mask_util.h \ google/protobuf/util/json_util.h \ + google/protobuf/util/zero_copy_sink.h \ google/protobuf/util/message_differencer.h \ google/protobuf/util/time_util.h \ google/protobuf/util/type_resolver.h \ @@ -311,6 +312,7 @@ libprotobuf_la_SOURCES = \ google/protobuf/util/internal/utility.cc \ google/protobuf/util/internal/utility.h \ google/protobuf/util/json_util.cc \ + google/protobuf/util/zero_copy_sink.cc \ google/protobuf/util/message_differencer.cc \ google/protobuf/util/time_util.cc \ google/protobuf/util/type_resolver_util.cc \ @@ -829,6 +831,7 @@ protobuf_test_SOURCES = \ google/protobuf/util/internal/protostream_objectwriter_test.cc \ google/protobuf/util/internal/type_info_test_helper.cc \ google/protobuf/util/json_util_test.cc \ + google/protobuf/util/zero_copy_sink_test.cc \ google/protobuf/util/message_differencer_unittest.cc \ google/protobuf/util/time_util_test.cc \ google/protobuf/util/type_resolver_util_test.cc \ diff --git a/src/file_lists.cmake b/src/file_lists.cmake index c849a6236e..32f10b2705 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -92,6 +92,7 @@ set(libprotobuf_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/util/message_differencer.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/util/time_util.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver_util.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/util/zero_copy_sink.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format_lite.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/wrappers.pb.cc @@ -213,6 +214,7 @@ set(libprotobuf_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/util/time_util.h ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver.h ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver_util.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/util/zero_copy_sink.h ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format.h ${protobuf_SOURCE_DIR}/src/google/protobuf/wire_format_lite.h ${protobuf_SOURCE_DIR}/src/google/protobuf/wrappers.pb.h @@ -843,6 +845,7 @@ set(util_test_files ${protobuf_SOURCE_DIR}/src/google/protobuf/util/message_differencer_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/util/time_util_test.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver_util_test.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/util/zero_copy_sink_test.cc ) # //src/google/protobuf/util:test_proto_srcs diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 6ba508a5f0..38ea055310 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -134,14 +134,6 @@ SerialArena::Memory SerialArena::Free(Deallocator deallocator) { return mem; } -PROTOBUF_NOINLINE -std::pair -SerialArena::AllocateAlignedWithCleanupFallback( - size_t n, const AllocationPolicy* policy) { - AllocateNewBlock(n + kCleanupSize, policy); - return AllocateFromExistingWithCleanupFallback(n); -} - PROTOBUF_NOINLINE void* SerialArena::AllocateAlignedFallback(size_t n, const AllocationPolicy* policy) { @@ -408,15 +400,16 @@ uint64_t ThreadSafeArena::Reset() { return space_allocated; } -std::pair -ThreadSafeArena::AllocateAlignedWithCleanup(size_t n, - const std::type_info* type) { +void* ThreadSafeArena::AllocateAlignedWithCleanup(size_t n, size_t align, + void (*destructor)(void*), + const std::type_info* type) { SerialArena* arena; if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && GetSerialArenaFast(&arena))) { - return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + return arena->AllocateAlignedWithCleanup(n, align, destructor, + alloc_policy_.get()); } else { - return AllocateAlignedWithCleanupFallback(n, type); + return AllocateAlignedWithCleanupFallback(n, align, destructor, type); } } @@ -443,18 +436,19 @@ void* ThreadSafeArena::AllocateAlignedFallback(size_t n, } PROTOBUF_NOINLINE -std::pair -ThreadSafeArena::AllocateAlignedWithCleanupFallback( - size_t n, const std::type_info* type) { +void* ThreadSafeArena::AllocateAlignedWithCleanupFallback( + size_t n, size_t align, void (*destructor)(void*), + const std::type_info* type) { if (alloc_policy_.should_record_allocs()) { - alloc_policy_.RecordAlloc(type, n); + alloc_policy_.RecordAlloc(type, internal::AlignUpTo(n, align)); SerialArena* arena; if (GetSerialArenaFast(&arena)) { - return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + return arena->AllocateAlignedWithCleanup(n, align, destructor, + alloc_policy_.get()); } } return GetSerialArenaFallback(&thread_cache()) - ->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + ->AllocateAlignedWithCleanup(n, align, destructor, alloc_policy_.get()); } uint64_t ThreadSafeArena::SpaceAllocated() const { @@ -526,9 +520,10 @@ void* Arena::AllocateAlignedWithHookForArray(size_t n, } PROTOBUF_FUNC_ALIGN(32) -std::pair -Arena::AllocateAlignedWithCleanup(size_t n, const std::type_info* type) { - return impl_.AllocateAlignedWithCleanup(n, type); +void* Arena::AllocateAlignedWithCleanup(size_t n, size_t align, + void (*destructor)(void*), + const std::type_info* type) { + return impl_.AllocateAlignedWithCleanup(n, align, destructor, type); } } // namespace protobuf diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 0f9f81366f..ee4b4db607 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -90,12 +90,6 @@ class RepeatedPtrFieldBase; // defined in repeated_ptr_field.h template class GenericTypeHandler; // defined in repeated_field.h -inline PROTOBUF_ALWAYS_INLINE -void* AlignTo(void* ptr, size_t align) { - return reinterpret_cast( - (reinterpret_cast(ptr) + align - 1) & (~align + 1)); -} - // Templated cleanup methods. template void arena_destruct_object(void* object) { @@ -420,7 +414,18 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { template class InternalHelper { private: - struct Rank1 {}; + // A SFINAE friendly trait that probes for `U` but always evalues to + // `Arena*`. + template + using EnableIfArena = + typename std::enable_if::value, Arena*>::type; + + // Rather than use SFINAE that must fully cover the space of options in a + // mutually exclusive fashion, we use implicit conversions to base classes + // to force an explicit ranking for our preferences. The lowest ranked + // version that compiles will be accepted. + struct Rank2 {}; + struct Rank1 : Rank2 {}; struct Rank0 : Rank1 {}; static Arena* GetOwningArena(const T* p) { @@ -429,7 +434,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { template static auto GetOwningArena(Rank0, const U* p) - -> decltype(p->GetOwningArena()) { + -> EnableIfArenaGetOwningArena())> { return p->GetOwningArena(); } @@ -440,35 +445,31 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { static void InternalSwap(T* a, T* b) { a->InternalSwap(b); } - static Arena* GetArenaForAllocationInternal( - const T* p, std::true_type /*is_derived_from*/) { - return p->GetArenaForAllocation(); + static Arena* GetArenaForAllocation(const T* p) { + return GetArenaForAllocation(Rank0{}, p); } - static Arena* GetArenaForAllocationInternal( - const T* p, std::false_type /*is_derived_from*/) { - return GetArenaForAllocationForNonMessage( - p, typename is_arena_constructable::type()); + static Arena* GetArena(const T* p) { + // Rather than replicate probing for `GetArena` with fallback to nullptr, + // we borrow the implementation of `GetArenaForAllocation` but skip + // `Rank0` which probes for `GetArenaForAllocation`. + return GetArenaForAllocation(Rank1{}, p); } - static Arena* GetArenaForAllocationForNonMessage( - const T* p, std::true_type /*is_arena_constructible*/) { - return p->GetArena(); - } - - static Arena* GetArenaForAllocationForNonMessage( - const T* p, std::false_type /*is_arena_constructible*/) { - return GetArenaForAllocationForNonMessageNonArenaConstructible( - p, typename has_get_arena::type()); + template + static auto GetArenaForAllocation(Rank0, const U* p) + -> EnableIfArenaGetArenaForAllocation())> { + return p->GetArenaForAllocation(); } - static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible( - const T* p, std::true_type /*has_get_arena*/) { + template + static auto GetArenaForAllocation(Rank1, const U* p) + -> EnableIfArenaGetArena())> { return p->GetArena(); } - static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible( - const T* /* p */, std::false_type /*has_get_arena*/) { + template + static Arena* GetArenaForAllocation(Rank2, const U* p) { return nullptr; } @@ -494,18 +495,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { sizeof(char)> is_arena_constructable; - template () - .GetArena())>::value, - int>::type = 0> - static char HasGetArena(decltype(&U::GetArena)); - template - static double HasGetArena(...); - - typedef std::integral_constant(nullptr)) == - sizeof(char)> - has_get_arena; template static T* Construct(void* ptr, Args&&... args) { @@ -516,8 +505,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { return new T(nullptr); } - static Arena* GetArena(const T* p) { return p->GetArena(); } - friend class Arena; friend class TestUtil::ReflectionTester; }; @@ -533,8 +520,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { // For internal use only. template static Arena* InternalGetArenaForAllocation(const T* p) { - return InternalHelper::GetArenaForAllocationInternal( - p, std::is_convertible()); + return InternalHelper::GetArenaForAllocation(p); } // Creates message-owned arena. For internal use only. @@ -566,9 +552,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { private: internal::ThreadSafeArena impl_; - template - struct has_get_arena : InternalHelper::has_get_arena {}; - // Constructor solely used by message-owned arena. inline Arena(internal::MessageOwned) : impl_(internal::MessageOwned{}) {} @@ -620,18 +603,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { if (destructor == nullptr) { return AllocateAlignedWithHook(size, align, type); } else { - if (align <= 8) { - auto res = AllocateAlignedWithCleanup(internal::AlignUpTo8(size), type); - res.second->elem = res.first; - res.second->cleanup = destructor; - return res.first; - } else { - auto res = AllocateAlignedWithCleanup(size + align - 8, type); - auto ptr = internal::AlignTo(res.first, align); - res.second->elem = ptr; - res.second->cleanup = destructor; - return ptr; - } + return AllocateAlignedWithCleanup(size, align, destructor, type); } } @@ -774,26 +746,10 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { // Implementation for GetArena(). Only message objects with // InternalArenaConstructable_ tags can be associated with an arena, and such // objects must implement a GetArena() method. - template ::value, int>::type = 0> + template PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { return InternalHelper::GetArena(value); } - template ::value && - has_get_arena::value, - int>::type = 0> - PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { - return value->GetArena(); - } - template ::value && - !has_get_arena::value, - int>::type = 0> - PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { - (void)value; - return nullptr; - } void* AllocateAlignedWithHookForArray(size_t n, size_t align, const std::type_info* type) { @@ -828,8 +784,9 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { void* AllocateAlignedNoHook(size_t n); void* AllocateAlignedWithHook(size_t n, const std::type_info* type); void* AllocateAlignedWithHookForArray(size_t n, const std::type_info* type); - std::pair - AllocateAlignedWithCleanup(size_t n, const std::type_info* type); + void* AllocateAlignedWithCleanup(size_t n, size_t align, + void (*destructor)(void*), + const std::type_info* type); template friend class internal::GenericTypeHandler; diff --git a/src/google/protobuf/arena_impl.h b/src/google/protobuf/arena_impl.h index 76727688b5..0bfceb413a 100644 --- a/src/google/protobuf/arena_impl.h +++ b/src/google/protobuf/arena_impl.h @@ -67,6 +67,24 @@ inline constexpr size_t AlignUpTo8(size_t n) { return (n + 7) & static_cast(-8); } +inline constexpr size_t AlignUpTo(size_t n, size_t a) { + // We are wasting space by over allocating align - 8 bytes. Compared to a + // dedicated function that takes current alignment in consideration. Such a + // scheme would only waste (align - 8)/2 bytes on average, but requires a + // dedicated function in the outline arena allocation functions. Possibly + // re-evaluate tradeoffs later. + return a <= 8 ? AlignUpTo8(n) : n + a - 8; +} + +inline PROTOBUF_ALWAYS_INLINE void* AlignTo(void* p, size_t a) { + if (a <= 8) { + return p; + } else { + auto u = reinterpret_cast(p); + return reinterpret_cast((u + a - 1) & (~a + 1)); + } +} + using LifecycleIdAtomic = uint64_t; // MetricsCollector collects stats for a particular arena. @@ -198,16 +216,10 @@ enum class AllocationClient { kDefault, kArray }; // This class manages // 1) Arena bump allocation + owning memory blocks. // 2) Maintaining a cleanup list. -// It delagetes the actual memory allocation back to ThreadSafeArena, which +// It delegates the actual memory allocation back to ThreadSafeArena, which // contains the information on block growth policy and backing memory allocation // used. class PROTOBUF_EXPORT SerialArena { - public: - struct Memory { - void* ptr; - size_t size; - }; - // Node contains the ptr of the object to be cleaned up and the associated // cleanup function ptr. struct CleanupNode { @@ -215,6 +227,12 @@ class PROTOBUF_EXPORT SerialArena { void (*cleanup)(void*); // Function pointer to the destructor or deleter. }; + public: + struct Memory { + void* ptr; + size_t size; + }; + void CleanupList(); uint64_t SpaceAllocated() const { return space_allocated_.load(std::memory_order_relaxed); @@ -270,11 +288,11 @@ class PROTOBUF_EXPORT SerialArena { private: void* AllocateFromExisting(size_t n) { - void* ret = ptr_; - ptr_ += n; #ifdef ADDRESS_SANITIZER - ASAN_UNPOISON_MEMORY_REGION(ret, n); + ASAN_UNPOISON_MEMORY_REGION(ptr_, n); #endif // ADDRESS_SANITIZER + void* ret = ptr_; + ptr_ += n; return ret; } @@ -332,36 +350,48 @@ class PROTOBUF_EXPORT SerialArena { return true; } - std::pair AllocateAlignedWithCleanup( - size_t n, const AllocationPolicy* policy) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. - if (PROTOBUF_PREDICT_FALSE(!HasSpace(n + kCleanupSize))) { - return AllocateAlignedWithCleanupFallback(n, policy); + void* AllocateAlignedWithCleanup(size_t n, size_t align, + void (*destructor)(void*), + const AllocationPolicy* policy) { + size_t required = AlignUpTo(n, align) + kCleanupSize; + if (PROTOBUF_PREDICT_FALSE(!HasSpace(required))) { + AllocateNewBlock(required, policy); } - return AllocateFromExistingWithCleanupFallback(n); + return AllocateFromExistingWithCleanupFallback(n, align, destructor); + } + + void AddCleanup(void* elem, void (*cleanup)(void*), + const AllocationPolicy* policy) { + if (PROTOBUF_PREDICT_FALSE(!HasSpace(kCleanupSize))) { + AllocateNewBlock(kCleanupSize, policy); + } + AddCleanupFromExisting(elem, cleanup); } private: - std::pair AllocateFromExistingWithCleanupFallback( - size_t n) { - void* ret = ptr_; - ptr_ += n; - limit_ -= kCleanupSize; + void* AllocateFromExistingWithCleanupFallback(size_t n, size_t align, + void (*destructor)(void*)) { + n = AlignUpTo(n, align); #ifdef ADDRESS_SANITIZER - ASAN_UNPOISON_MEMORY_REGION(ret, n); - ASAN_UNPOISON_MEMORY_REGION(limit_, kCleanupSize); + ASAN_UNPOISON_MEMORY_REGION(ptr_, n); #endif // ADDRESS_SANITIZER - return CreatePair(ret, reinterpret_cast(limit_)); + void* ret = internal::AlignTo(ptr_, align); + ptr_ += n; + AddCleanupFromExisting(ret, destructor); + return ret; } - public: - void AddCleanup(void* elem, void (*cleanup)(void*), - const AllocationPolicy* policy) { - auto res = AllocateAlignedWithCleanup(0, policy); - res.second->elem = elem; - res.second->cleanup = cleanup; + void AddCleanupFromExisting(void* elem, void (*cleanup)(void*)) { +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(limit_ - kCleanupSize, kCleanupSize); +#endif // ADDRESS_SANITIZER + limit_ -= kCleanupSize; + auto c = reinterpret_cast(limit_); + c->elem = elem; + c->cleanup = cleanup; } + public: void* owner() const { return owner_; } SerialArena* next() const { return next_; } void set_next(SerialArena* next) { next_ = next; } @@ -427,14 +457,8 @@ class PROTOBUF_EXPORT SerialArena { // Constructor is private as only New() should be used. inline SerialArena(Block* b, void* owner, ThreadSafeArenaStats* stats); void* AllocateAlignedFallback(size_t n, const AllocationPolicy* policy); - std::pair AllocateAlignedWithCleanupFallback( - size_t n, const AllocationPolicy* policy); void AllocateNewBlock(size_t n, const AllocationPolicy* policy); - std::pair CreatePair(void* ptr, CleanupNode* node) { - return {ptr, node}; - } - public: static constexpr size_t kBlockHeaderSize = AlignUpTo8(sizeof(Block)); static constexpr size_t kCleanupSize = AlignUpTo8(sizeof(CleanupNode)); @@ -469,6 +493,13 @@ class PROTOBUF_EXPORT ThreadSafeArena { InitializeWithPolicy(mem, size, policy); } + // All protos have pointers back to the arena hence Arena must have + // pointer stability. + ThreadSafeArena(const ThreadSafeArena&) = delete; + ThreadSafeArena& operator=(const ThreadSafeArena&) = delete; + ThreadSafeArena(ThreadSafeArena&&) = delete; + ThreadSafeArena& operator=(ThreadSafeArena&&) = delete; + // Destructor deletes all owned heap allocated objects, and destructs objects // that have non-trivial destructors, except for proto2 message objects whose // destructors can be skipped. Also, frees all blocks except the initial block @@ -512,8 +543,9 @@ class PROTOBUF_EXPORT ThreadSafeArena { return false; } - std::pair AllocateAlignedWithCleanup( - size_t n, const std::type_info* type); + void* AllocateAlignedWithCleanup(size_t n, size_t align, + void (*destructor)(void*), + const std::type_info* type); // Add object pointer and cleanup function pointer to the list. void AddCleanup(void* elem, void (*cleanup)(void*)); @@ -541,8 +573,9 @@ class PROTOBUF_EXPORT ThreadSafeArena { void InitializeFrom(void* mem, size_t size); void InitializeWithPolicy(void* mem, size_t size, AllocationPolicy policy); void* AllocateAlignedFallback(size_t n, const std::type_info* type); - std::pair - AllocateAlignedWithCleanupFallback(size_t n, const std::type_info* type); + void* AllocateAlignedWithCleanupFallback(size_t n, size_t align, + void (*destructor)(void*), + const std::type_info* type); void Init(); void SetInitialBlock(void* mem, size_t size); @@ -659,12 +692,6 @@ class PROTOBUF_EXPORT ThreadSafeArena { ThreadSafeArenaStatsHandle arena_stats_; - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ThreadSafeArena); - // All protos have pointers back to the arena hence Arena must have - // pointer stability. - ThreadSafeArena(ThreadSafeArena&&) = delete; - ThreadSafeArena& operator=(ThreadSafeArena&&) = delete; - public: // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 // to protect the invariant that pos is always at a multiple of 8. diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc index 65da0d3d4d..03b1438b44 100644 --- a/src/google/protobuf/arenastring.cc +++ b/src/google/protobuf/arenastring.cc @@ -125,8 +125,7 @@ void ArenaStringPtr::Set(ConstStringParam value, Arena* arena) { } else { #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (arena == nullptr) { - GOOGLE_DCHECK(tagged_ptr_.IsAllocated()); - auto* old = tagged_ptr_.Get(); + auto* old = tagged_ptr_.GetIfAllocated(); tagged_ptr_ = CreateString(value); delete old; } else { diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 5e9a2c4185..a688bcfb06 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -1276,6 +1276,7 @@ bool CommandLineInterface::ParseInputFiles( } parsed_files->push_back(parsed_file); + // Enforce --disallow_services. if (disallow_services_ && parsed_file->service_count() > 0) { std::cerr << parsed_file->name() diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h index e8425508b1..8064b397f8 100644 --- a/src/google/protobuf/compiler/command_line_interface.h +++ b/src/google/protobuf/compiler/command_line_interface.h @@ -398,7 +398,7 @@ class PROTOC_EXPORT CommandLineInterface { // True if we should treat warnings as errors that fail the compilation. bool fatal_warnings_ = false; - std::vector > + std::vector> proto_path_; // Search path for proto files. std::vector input_files_; // Names of the input proto files. diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index 5bd37d147b..f220fad8be 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -634,14 +634,17 @@ bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { root_location.RecordLegacyLocation(file, DescriptorPool::ErrorCollector::OTHER); - if (require_syntax_identifier_ || LookingAt("syntax")) { + if (require_syntax_identifier_ || LookingAt("syntax") + ) { if (!ParseSyntaxIdentifier(root_location)) { // Don't attempt to parse the file if we didn't recognize the syntax // identifier. return false; } // Store the syntax into the file. - if (file != nullptr) file->set_syntax(syntax_identifier_); + if (file != nullptr) { + file->set_syntax(syntax_identifier_); + } } else if (!stop_after_syntax_identifier_) { GOOGLE_LOG(WARNING) << "No syntax specified for the proto file: " << file->name() << ". Please use 'syntax = \"proto2\";' " @@ -678,9 +681,10 @@ bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { bool Parser::ParseSyntaxIdentifier(const LocationRecorder& parent) { LocationRecorder syntax_location(parent, FileDescriptorProto::kSyntaxFieldNumber); - DO(Consume( - "syntax", - "File must begin with a syntax statement, e.g. 'syntax = \"proto2\";'.")); + DO(Consume("syntax", + "File must begin with a syntax statement, e.g. 'syntax = " + "\"proto2\";'.")); + DO(Consume("=")); io::Tokenizer::Token syntax_token = input_->current(); std::string syntax; @@ -688,7 +692,6 @@ bool Parser::ParseSyntaxIdentifier(const LocationRecorder& parent) { DO(ConsumeEndOfDeclaration(";", &syntax_location)); syntax_identifier_ = syntax; - if (syntax != "proto2" && syntax != "proto3" && !stop_after_syntax_identifier_) { AddError(syntax_token.line, syntax_token.column, diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 2d681d957f..55ed7acb6f 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -264,6 +264,7 @@ TEST_F(ParserTest, WarnIfFieldNameContainsNumberImmediatelyFollowUnderscore) { "song_name_1.") != std::string::npos); } + // =================================================================== typedef ParserTest ParseMessageTest; diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 5f3427dc72..139e6f892f 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -2562,7 +2562,9 @@ void FileDescriptor::CopyTo(FileDescriptorProto* proto) const { proto->set_name(name()); if (!package().empty()) proto->set_package(package()); // TODO(liujisi): Also populate when syntax="proto2". - if (syntax() == SYNTAX_PROTO3) proto->set_syntax(SyntaxName(syntax())); + if (syntax() == SYNTAX_PROTO3 + ) proto->set_syntax(SyntaxName(syntax())); + for (int i = 0; i < dependency_count(); i++) { proto->add_dependency(dependency(i)->name()); @@ -4982,7 +4984,8 @@ static void PlanAllocationSize(const FileDescriptorProto& proto, internal::FlatAllocator& alloc) { alloc.PlanArray(1); alloc.PlanArray(1); - alloc.PlanArray(2); // name + package + alloc.PlanArray(2 + ); // name + package if (proto.has_options()) alloc.PlanArray(1); if (proto.has_source_code_info()) alloc.PlanArray(1); diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 1b8728ec63..685e9235d8 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -54,6 +54,9 @@ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_H__ #define GOOGLE_PROTOBUF_DESCRIPTOR_H__ +#include + +#include #include #include @@ -1702,7 +1705,7 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FileDescriptor); }; -PROTOBUF_INTERNAL_CHECK_CLASS_SIZE(FileDescriptor, 144); +PROTOBUF_INTERNAL_CHECK_CLASS_SIZE(FileDescriptor, 152); // =================================================================== diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index c77c83c1e9..2dfece5b61 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -48,6 +48,7 @@ PROTOBUF_CONSTEXPR FileDescriptorProto::FileDescriptorProto( , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.package_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.syntax_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} + , /*decltype(_impl_.edition_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.options_)*/nullptr , /*decltype(_impl_.source_code_info_)*/nullptr} {} struct FileDescriptorProtoDefaultTypeInternal { @@ -540,6 +541,7 @@ const uint32_t TableStruct_google_2fprotobuf_2fdescriptor_2eproto::offsets[] PRO PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::FileDescriptorProto, _impl_.options_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::FileDescriptorProto, _impl_.source_code_info_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::FileDescriptorProto, _impl_.syntax_), + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::FileDescriptorProto, _impl_.edition_), 0, 1, ~0u, @@ -549,9 +551,10 @@ const uint32_t TableStruct_google_2fprotobuf_2fdescriptor_2eproto::offsets[] PRO ~0u, ~0u, ~0u, - 3, 4, + 5, 2, + 3, PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::DescriptorProto_ExtensionRange, _impl_._has_bits_), PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::DescriptorProto_ExtensionRange, _internal_metadata_), ~0u, // no _extensions_ @@ -977,32 +980,32 @@ const uint32_t TableStruct_google_2fprotobuf_2fdescriptor_2eproto::offsets[] PRO }; static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FileDescriptorSet)}, - { 9, 29, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FileDescriptorProto)}, - { 41, 52, -1, sizeof(::PROTOBUF_NAMESPACE_ID::DescriptorProto_ExtensionRange)}, - { 55, 65, -1, sizeof(::PROTOBUF_NAMESPACE_ID::DescriptorProto_ReservedRange)}, - { 67, 85, -1, sizeof(::PROTOBUF_NAMESPACE_ID::DescriptorProto)}, - { 95, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::ExtensionRangeOptions)}, - { 104, 123, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FieldDescriptorProto)}, - { 134, 144, -1, sizeof(::PROTOBUF_NAMESPACE_ID::OneofDescriptorProto)}, - { 146, 156, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumDescriptorProto_EnumReservedRange)}, - { 158, 171, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumDescriptorProto)}, - { 176, 187, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumValueDescriptorProto)}, - { 190, 201, -1, sizeof(::PROTOBUF_NAMESPACE_ID::ServiceDescriptorProto)}, - { 204, 218, -1, sizeof(::PROTOBUF_NAMESPACE_ID::MethodDescriptorProto)}, - { 224, 253, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FileOptions)}, - { 274, 287, -1, sizeof(::PROTOBUF_NAMESPACE_ID::MessageOptions)}, - { 292, 308, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FieldOptions)}, - { 316, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::OneofOptions)}, - { 325, 336, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumOptions)}, - { 339, 349, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumValueOptions)}, - { 351, 361, -1, sizeof(::PROTOBUF_NAMESPACE_ID::ServiceOptions)}, - { 363, 374, -1, sizeof(::PROTOBUF_NAMESPACE_ID::MethodOptions)}, - { 377, 387, -1, sizeof(::PROTOBUF_NAMESPACE_ID::UninterpretedOption_NamePart)}, - { 389, 404, -1, sizeof(::PROTOBUF_NAMESPACE_ID::UninterpretedOption)}, - { 411, 424, -1, sizeof(::PROTOBUF_NAMESPACE_ID::SourceCodeInfo_Location)}, - { 429, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::SourceCodeInfo)}, - { 438, 450, -1, sizeof(::PROTOBUF_NAMESPACE_ID::GeneratedCodeInfo_Annotation)}, - { 454, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::GeneratedCodeInfo)}, + { 9, 30, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FileDescriptorProto)}, + { 43, 54, -1, sizeof(::PROTOBUF_NAMESPACE_ID::DescriptorProto_ExtensionRange)}, + { 57, 67, -1, sizeof(::PROTOBUF_NAMESPACE_ID::DescriptorProto_ReservedRange)}, + { 69, 87, -1, sizeof(::PROTOBUF_NAMESPACE_ID::DescriptorProto)}, + { 97, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::ExtensionRangeOptions)}, + { 106, 125, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FieldDescriptorProto)}, + { 136, 146, -1, sizeof(::PROTOBUF_NAMESPACE_ID::OneofDescriptorProto)}, + { 148, 158, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumDescriptorProto_EnumReservedRange)}, + { 160, 173, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumDescriptorProto)}, + { 178, 189, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumValueDescriptorProto)}, + { 192, 203, -1, sizeof(::PROTOBUF_NAMESPACE_ID::ServiceDescriptorProto)}, + { 206, 220, -1, sizeof(::PROTOBUF_NAMESPACE_ID::MethodDescriptorProto)}, + { 226, 255, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FileOptions)}, + { 276, 289, -1, sizeof(::PROTOBUF_NAMESPACE_ID::MessageOptions)}, + { 294, 310, -1, sizeof(::PROTOBUF_NAMESPACE_ID::FieldOptions)}, + { 318, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::OneofOptions)}, + { 327, 338, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumOptions)}, + { 341, 351, -1, sizeof(::PROTOBUF_NAMESPACE_ID::EnumValueOptions)}, + { 353, 363, -1, sizeof(::PROTOBUF_NAMESPACE_ID::ServiceOptions)}, + { 365, 376, -1, sizeof(::PROTOBUF_NAMESPACE_ID::MethodOptions)}, + { 379, 389, -1, sizeof(::PROTOBUF_NAMESPACE_ID::UninterpretedOption_NamePart)}, + { 391, 406, -1, sizeof(::PROTOBUF_NAMESPACE_ID::UninterpretedOption)}, + { 413, 426, -1, sizeof(::PROTOBUF_NAMESPACE_ID::SourceCodeInfo_Location)}, + { 431, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::SourceCodeInfo)}, + { 440, 452, -1, sizeof(::PROTOBUF_NAMESPACE_ID::GeneratedCodeInfo_Annotation)}, + { 456, -1, -1, sizeof(::PROTOBUF_NAMESPACE_ID::GeneratedCodeInfo)}, }; static const ::_pb::Message* const file_default_instances[] = { @@ -1039,7 +1042,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto[] PR "\n google/protobuf/descriptor.proto\022\017goog" "le.protobuf\"G\n\021FileDescriptorSet\0222\n\004file" "\030\001 \003(\0132$.google.protobuf.FileDescriptorP" - "roto\"\333\003\n\023FileDescriptorProto\022\014\n\004name\030\001 \001" + "roto\"\354\003\n\023FileDescriptorProto\022\014\n\004name\030\001 \001" "(\t\022\017\n\007package\030\002 \001(\t\022\022\n\ndependency\030\003 \003(\t\022" "\031\n\021public_dependency\030\n \003(\005\022\027\n\017weak_depen" "dency\030\013 \003(\005\0226\n\014message_type\030\004 \003(\0132 .goog" @@ -1051,147 +1054,148 @@ const char descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto[] PR "\022-\n\007options\030\010 \001(\0132\034.google.protobuf.File" "Options\0229\n\020source_code_info\030\t \001(\0132\037.goog" "le.protobuf.SourceCodeInfo\022\016\n\006syntax\030\014 \001" - "(\t\"\251\005\n\017DescriptorProto\022\014\n\004name\030\001 \001(\t\0224\n\005" - "field\030\002 \003(\0132%.google.protobuf.FieldDescr" - "iptorProto\0228\n\textension\030\006 \003(\0132%.google.p" - "rotobuf.FieldDescriptorProto\0225\n\013nested_t" - "ype\030\003 \003(\0132 .google.protobuf.DescriptorPr" - "oto\0227\n\tenum_type\030\004 \003(\0132$.google.protobuf" - ".EnumDescriptorProto\022H\n\017extension_range\030" - "\005 \003(\0132/.google.protobuf.DescriptorProto." - "ExtensionRange\0229\n\noneof_decl\030\010 \003(\0132%.goo" - "gle.protobuf.OneofDescriptorProto\0220\n\007opt" - "ions\030\007 \001(\0132\037.google.protobuf.MessageOpti" - "ons\022F\n\016reserved_range\030\t \003(\0132..google.pro" - "tobuf.DescriptorProto.ReservedRange\022\025\n\rr" - "eserved_name\030\n \003(\t\032e\n\016ExtensionRange\022\r\n\005" - "start\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\0227\n\007options\030\003 \001(" - "\0132&.google.protobuf.ExtensionRangeOption" - "s\032+\n\rReservedRange\022\r\n\005start\030\001 \001(\005\022\013\n\003end" - "\030\002 \001(\005\"g\n\025ExtensionRangeOptions\022C\n\024unint" - "erpreted_option\030\347\007 \003(\0132$.google.protobuf" - ".UninterpretedOption*\t\010\350\007\020\200\200\200\200\002\"\325\005\n\024Fiel" - "dDescriptorProto\022\014\n\004name\030\001 \001(\t\022\016\n\006number" - "\030\003 \001(\005\022:\n\005label\030\004 \001(\0162+.google.protobuf." - "FieldDescriptorProto.Label\0228\n\004type\030\005 \001(\016" - "2*.google.protobuf.FieldDescriptorProto." - "Type\022\021\n\ttype_name\030\006 \001(\t\022\020\n\010extendee\030\002 \001(" - "\t\022\025\n\rdefault_value\030\007 \001(\t\022\023\n\013oneof_index\030" - "\t \001(\005\022\021\n\tjson_name\030\n \001(\t\022.\n\007options\030\010 \001(" - "\0132\035.google.protobuf.FieldOptions\022\027\n\017prot" - "o3_optional\030\021 \001(\010\"\266\002\n\004Type\022\017\n\013TYPE_DOUBL" - "E\020\001\022\016\n\nTYPE_FLOAT\020\002\022\016\n\nTYPE_INT64\020\003\022\017\n\013T" - "YPE_UINT64\020\004\022\016\n\nTYPE_INT32\020\005\022\020\n\014TYPE_FIX" - "ED64\020\006\022\020\n\014TYPE_FIXED32\020\007\022\r\n\tTYPE_BOOL\020\010\022" - "\017\n\013TYPE_STRING\020\t\022\016\n\nTYPE_GROUP\020\n\022\020\n\014TYPE" - "_MESSAGE\020\013\022\016\n\nTYPE_BYTES\020\014\022\017\n\013TYPE_UINT3" - "2\020\r\022\r\n\tTYPE_ENUM\020\016\022\021\n\rTYPE_SFIXED32\020\017\022\021\n" - "\rTYPE_SFIXED64\020\020\022\017\n\013TYPE_SINT32\020\021\022\017\n\013TYP" - "E_SINT64\020\022\"C\n\005Label\022\022\n\016LABEL_OPTIONAL\020\001\022" - "\022\n\016LABEL_REQUIRED\020\002\022\022\n\016LABEL_REPEATED\020\003\"" - "T\n\024OneofDescriptorProto\022\014\n\004name\030\001 \001(\t\022.\n" - "\007options\030\002 \001(\0132\035.google.protobuf.OneofOp" - "tions\"\244\002\n\023EnumDescriptorProto\022\014\n\004name\030\001 " - "\001(\t\0228\n\005value\030\002 \003(\0132).google.protobuf.Enu" - "mValueDescriptorProto\022-\n\007options\030\003 \001(\0132\034" - ".google.protobuf.EnumOptions\022N\n\016reserved" - "_range\030\004 \003(\01326.google.protobuf.EnumDescr" - "iptorProto.EnumReservedRange\022\025\n\rreserved" - "_name\030\005 \003(\t\032/\n\021EnumReservedRange\022\r\n\005star" - "t\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\"l\n\030EnumValueDescrip" - "torProto\022\014\n\004name\030\001 \001(\t\022\016\n\006number\030\002 \001(\005\0222" - "\n\007options\030\003 \001(\0132!.google.protobuf.EnumVa" - "lueOptions\"\220\001\n\026ServiceDescriptorProto\022\014\n" - "\004name\030\001 \001(\t\0226\n\006method\030\002 \003(\0132&.google.pro" - "tobuf.MethodDescriptorProto\0220\n\007options\030\003" - " \001(\0132\037.google.protobuf.ServiceOptions\"\301\001" - "\n\025MethodDescriptorProto\022\014\n\004name\030\001 \001(\t\022\022\n" - "\ninput_type\030\002 \001(\t\022\023\n\013output_type\030\003 \001(\t\022/" - "\n\007options\030\004 \001(\0132\036.google.protobuf.Method" - "Options\022\037\n\020client_streaming\030\005 \001(\010:\005false" - "\022\037\n\020server_streaming\030\006 \001(\010:\005false\"\245\006\n\013Fi" - "leOptions\022\024\n\014java_package\030\001 \001(\t\022\034\n\024java_" - "outer_classname\030\010 \001(\t\022\"\n\023java_multiple_f" - "iles\030\n \001(\010:\005false\022)\n\035java_generate_equal" - "s_and_hash\030\024 \001(\010B\002\030\001\022%\n\026java_string_chec" - "k_utf8\030\033 \001(\010:\005false\022F\n\014optimize_for\030\t \001(" - "\0162).google.protobuf.FileOptions.Optimize" - "Mode:\005SPEED\022\022\n\ngo_package\030\013 \001(\t\022\"\n\023cc_ge" - "neric_services\030\020 \001(\010:\005false\022$\n\025java_gene" - "ric_services\030\021 \001(\010:\005false\022\"\n\023py_generic_" - "services\030\022 \001(\010:\005false\022#\n\024php_generic_ser" - "vices\030* \001(\010:\005false\022\031\n\ndeprecated\030\027 \001(\010:\005" - "false\022\036\n\020cc_enable_arenas\030\037 \001(\010:\004true\022\031\n" - "\021objc_class_prefix\030$ \001(\t\022\030\n\020csharp_names" - "pace\030% \001(\t\022\024\n\014swift_prefix\030\' \001(\t\022\030\n\020php_" - "class_prefix\030( \001(\t\022\025\n\rphp_namespace\030) \001(" - "\t\022\036\n\026php_metadata_namespace\030, \001(\t\022\024\n\014rub" - "y_package\030- \001(\t\022C\n\024uninterpreted_option\030" - "\347\007 \003(\0132$.google.protobuf.UninterpretedOp" - "tion\":\n\014OptimizeMode\022\t\n\005SPEED\020\001\022\r\n\tCODE_" - "SIZE\020\002\022\020\n\014LITE_RUNTIME\020\003*\t\010\350\007\020\200\200\200\200\002J\004\010&\020" - "\'\"\204\002\n\016MessageOptions\022&\n\027message_set_wire" - "_format\030\001 \001(\010:\005false\022.\n\037no_standard_desc" - "riptor_accessor\030\002 \001(\010:\005false\022\031\n\ndeprecat" - "ed\030\003 \001(\010:\005false\022\021\n\tmap_entry\030\007 \001(\010\022C\n\024un" - "interpreted_option\030\347\007 \003(\0132$.google.proto" - "buf.UninterpretedOption*\t\010\350\007\020\200\200\200\200\002J\004\010\004\020\005" - "J\004\010\005\020\006J\004\010\006\020\007J\004\010\010\020\tJ\004\010\t\020\n\"\276\003\n\014FieldOption" - "s\022:\n\005ctype\030\001 \001(\0162#.google.protobuf.Field" - "Options.CType:\006STRING\022\016\n\006packed\030\002 \001(\010\022\?\n" - "\006jstype\030\006 \001(\0162$.google.protobuf.FieldOpt" - "ions.JSType:\tJS_NORMAL\022\023\n\004lazy\030\005 \001(\010:\005fa" - "lse\022\036\n\017unverified_lazy\030\017 \001(\010:\005false\022\031\n\nd" - "eprecated\030\003 \001(\010:\005false\022\023\n\004weak\030\n \001(\010:\005fa" - "lse\022C\n\024uninterpreted_option\030\347\007 \003(\0132$.goo" - "gle.protobuf.UninterpretedOption\"/\n\005CTyp" - "e\022\n\n\006STRING\020\000\022\010\n\004CORD\020\001\022\020\n\014STRING_PIECE\020" - "\002\"5\n\006JSType\022\r\n\tJS_NORMAL\020\000\022\r\n\tJS_STRING\020" - "\001\022\r\n\tJS_NUMBER\020\002*\t\010\350\007\020\200\200\200\200\002J\004\010\004\020\005\"^\n\014One" - "ofOptions\022C\n\024uninterpreted_option\030\347\007 \003(\013" - "2$.google.protobuf.UninterpretedOption*\t" - "\010\350\007\020\200\200\200\200\002\"\223\001\n\013EnumOptions\022\023\n\013allow_alias" - "\030\002 \001(\010\022\031\n\ndeprecated\030\003 \001(\010:\005false\022C\n\024uni" - "nterpreted_option\030\347\007 \003(\0132$.google.protob" - "uf.UninterpretedOption*\t\010\350\007\020\200\200\200\200\002J\004\010\005\020\006\"" - "}\n\020EnumValueOptions\022\031\n\ndeprecated\030\001 \001(\010:" - "\005false\022C\n\024uninterpreted_option\030\347\007 \003(\0132$." - "google.protobuf.UninterpretedOption*\t\010\350\007" - "\020\200\200\200\200\002\"{\n\016ServiceOptions\022\031\n\ndeprecated\030!" - " \001(\010:\005false\022C\n\024uninterpreted_option\030\347\007 \003" + "(\t\022\017\n\007edition\030\r \001(\t\"\251\005\n\017DescriptorProto\022" + "\014\n\004name\030\001 \001(\t\0224\n\005field\030\002 \003(\0132%.google.pr" + "otobuf.FieldDescriptorProto\0228\n\textension" + "\030\006 \003(\0132%.google.protobuf.FieldDescriptor" + "Proto\0225\n\013nested_type\030\003 \003(\0132 .google.prot" + "obuf.DescriptorProto\0227\n\tenum_type\030\004 \003(\0132" + "$.google.protobuf.EnumDescriptorProto\022H\n" + "\017extension_range\030\005 \003(\0132/.google.protobuf" + ".DescriptorProto.ExtensionRange\0229\n\noneof" + "_decl\030\010 \003(\0132%.google.protobuf.OneofDescr" + "iptorProto\0220\n\007options\030\007 \001(\0132\037.google.pro" + "tobuf.MessageOptions\022F\n\016reserved_range\030\t" + " \003(\0132..google.protobuf.DescriptorProto.R" + "eservedRange\022\025\n\rreserved_name\030\n \003(\t\032e\n\016E" + "xtensionRange\022\r\n\005start\030\001 \001(\005\022\013\n\003end\030\002 \001(" + "\005\0227\n\007options\030\003 \001(\0132&.google.protobuf.Ext" + "ensionRangeOptions\032+\n\rReservedRange\022\r\n\005s" + "tart\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\"g\n\025ExtensionRang" + "eOptions\022C\n\024uninterpreted_option\030\347\007 \003(\0132" + "$.google.protobuf.UninterpretedOption*\t\010" + "\350\007\020\200\200\200\200\002\"\325\005\n\024FieldDescriptorProto\022\014\n\004nam" + "e\030\001 \001(\t\022\016\n\006number\030\003 \001(\005\022:\n\005label\030\004 \001(\0162+" + ".google.protobuf.FieldDescriptorProto.La" + "bel\0228\n\004type\030\005 \001(\0162*.google.protobuf.Fiel" + "dDescriptorProto.Type\022\021\n\ttype_name\030\006 \001(\t" + "\022\020\n\010extendee\030\002 \001(\t\022\025\n\rdefault_value\030\007 \001(" + "\t\022\023\n\013oneof_index\030\t \001(\005\022\021\n\tjson_name\030\n \001(" + "\t\022.\n\007options\030\010 \001(\0132\035.google.protobuf.Fie" + "ldOptions\022\027\n\017proto3_optional\030\021 \001(\010\"\266\002\n\004T" + "ype\022\017\n\013TYPE_DOUBLE\020\001\022\016\n\nTYPE_FLOAT\020\002\022\016\n\n" + "TYPE_INT64\020\003\022\017\n\013TYPE_UINT64\020\004\022\016\n\nTYPE_IN" + "T32\020\005\022\020\n\014TYPE_FIXED64\020\006\022\020\n\014TYPE_FIXED32\020" + "\007\022\r\n\tTYPE_BOOL\020\010\022\017\n\013TYPE_STRING\020\t\022\016\n\nTYP" + "E_GROUP\020\n\022\020\n\014TYPE_MESSAGE\020\013\022\016\n\nTYPE_BYTE" + "S\020\014\022\017\n\013TYPE_UINT32\020\r\022\r\n\tTYPE_ENUM\020\016\022\021\n\rT" + "YPE_SFIXED32\020\017\022\021\n\rTYPE_SFIXED64\020\020\022\017\n\013TYP" + "E_SINT32\020\021\022\017\n\013TYPE_SINT64\020\022\"C\n\005Label\022\022\n\016" + "LABEL_OPTIONAL\020\001\022\022\n\016LABEL_REQUIRED\020\002\022\022\n\016" + "LABEL_REPEATED\020\003\"T\n\024OneofDescriptorProto" + "\022\014\n\004name\030\001 \001(\t\022.\n\007options\030\002 \001(\0132\035.google" + ".protobuf.OneofOptions\"\244\002\n\023EnumDescripto" + "rProto\022\014\n\004name\030\001 \001(\t\0228\n\005value\030\002 \003(\0132).go" + "ogle.protobuf.EnumValueDescriptorProto\022-" + "\n\007options\030\003 \001(\0132\034.google.protobuf.EnumOp" + "tions\022N\n\016reserved_range\030\004 \003(\01326.google.p" + "rotobuf.EnumDescriptorProto.EnumReserved" + "Range\022\025\n\rreserved_name\030\005 \003(\t\032/\n\021EnumRese" + "rvedRange\022\r\n\005start\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\"l\n" + "\030EnumValueDescriptorProto\022\014\n\004name\030\001 \001(\t\022" + "\016\n\006number\030\002 \001(\005\0222\n\007options\030\003 \001(\0132!.googl" + "e.protobuf.EnumValueOptions\"\220\001\n\026ServiceD" + "escriptorProto\022\014\n\004name\030\001 \001(\t\0226\n\006method\030\002" + " \003(\0132&.google.protobuf.MethodDescriptorP" + "roto\0220\n\007options\030\003 \001(\0132\037.google.protobuf." + "ServiceOptions\"\301\001\n\025MethodDescriptorProto" + "\022\014\n\004name\030\001 \001(\t\022\022\n\ninput_type\030\002 \001(\t\022\023\n\013ou" + "tput_type\030\003 \001(\t\022/\n\007options\030\004 \001(\0132\036.googl" + "e.protobuf.MethodOptions\022\037\n\020client_strea" + "ming\030\005 \001(\010:\005false\022\037\n\020server_streaming\030\006 " + "\001(\010:\005false\"\245\006\n\013FileOptions\022\024\n\014java_packa" + "ge\030\001 \001(\t\022\034\n\024java_outer_classname\030\010 \001(\t\022\"" + "\n\023java_multiple_files\030\n \001(\010:\005false\022)\n\035ja" + "va_generate_equals_and_hash\030\024 \001(\010B\002\030\001\022%\n" + "\026java_string_check_utf8\030\033 \001(\010:\005false\022F\n\014" + "optimize_for\030\t \001(\0162).google.protobuf.Fil" + "eOptions.OptimizeMode:\005SPEED\022\022\n\ngo_packa" + "ge\030\013 \001(\t\022\"\n\023cc_generic_services\030\020 \001(\010:\005f" + "alse\022$\n\025java_generic_services\030\021 \001(\010:\005fal" + "se\022\"\n\023py_generic_services\030\022 \001(\010:\005false\022#" + "\n\024php_generic_services\030* \001(\010:\005false\022\031\n\nd" + "eprecated\030\027 \001(\010:\005false\022\036\n\020cc_enable_aren" + "as\030\037 \001(\010:\004true\022\031\n\021objc_class_prefix\030$ \001(" + "\t\022\030\n\020csharp_namespace\030% \001(\t\022\024\n\014swift_pre" + "fix\030\' \001(\t\022\030\n\020php_class_prefix\030( \001(\t\022\025\n\rp" + "hp_namespace\030) \001(\t\022\036\n\026php_metadata_names" + "pace\030, \001(\t\022\024\n\014ruby_package\030- \001(\t\022C\n\024unin" + "terpreted_option\030\347\007 \003(\0132$.google.protobu" + "f.UninterpretedOption\":\n\014OptimizeMode\022\t\n" + "\005SPEED\020\001\022\r\n\tCODE_SIZE\020\002\022\020\n\014LITE_RUNTIME\020" + "\003*\t\010\350\007\020\200\200\200\200\002J\004\010&\020\'\"\204\002\n\016MessageOptions\022&\n" + "\027message_set_wire_format\030\001 \001(\010:\005false\022.\n" + "\037no_standard_descriptor_accessor\030\002 \001(\010:\005" + "false\022\031\n\ndeprecated\030\003 \001(\010:\005false\022\021\n\tmap_" + "entry\030\007 \001(\010\022C\n\024uninterpreted_option\030\347\007 \003" "(\0132$.google.protobuf.UninterpretedOption" - "*\t\010\350\007\020\200\200\200\200\002\"\255\002\n\rMethodOptions\022\031\n\ndepreca" - "ted\030! \001(\010:\005false\022_\n\021idempotency_level\030\" " - "\001(\0162/.google.protobuf.MethodOptions.Idem" - "potencyLevel:\023IDEMPOTENCY_UNKNOWN\022C\n\024uni" - "nterpreted_option\030\347\007 \003(\0132$.google.protob" - "uf.UninterpretedOption\"P\n\020IdempotencyLev" - "el\022\027\n\023IDEMPOTENCY_UNKNOWN\020\000\022\023\n\017NO_SIDE_E" - "FFECTS\020\001\022\016\n\nIDEMPOTENT\020\002*\t\010\350\007\020\200\200\200\200\002\"\236\002\n\023" - "UninterpretedOption\022;\n\004name\030\002 \003(\0132-.goog" - "le.protobuf.UninterpretedOption.NamePart" - "\022\030\n\020identifier_value\030\003 \001(\t\022\032\n\022positive_i" - "nt_value\030\004 \001(\004\022\032\n\022negative_int_value\030\005 \001" - "(\003\022\024\n\014double_value\030\006 \001(\001\022\024\n\014string_value" - "\030\007 \001(\014\022\027\n\017aggregate_value\030\010 \001(\t\0323\n\010NameP" - "art\022\021\n\tname_part\030\001 \002(\t\022\024\n\014is_extension\030\002" - " \002(\010\"\325\001\n\016SourceCodeInfo\022:\n\010location\030\001 \003(" - "\0132(.google.protobuf.SourceCodeInfo.Locat" - "ion\032\206\001\n\010Location\022\020\n\004path\030\001 \003(\005B\002\020\001\022\020\n\004sp" - "an\030\002 \003(\005B\002\020\001\022\030\n\020leading_comments\030\003 \001(\t\022\031" - "\n\021trailing_comments\030\004 \001(\t\022!\n\031leading_det" - "ached_comments\030\006 \003(\t\"\247\001\n\021GeneratedCodeIn" - "fo\022A\n\nannotation\030\001 \003(\0132-.google.protobuf" - ".GeneratedCodeInfo.Annotation\032O\n\nAnnotat" - "ion\022\020\n\004path\030\001 \003(\005B\002\020\001\022\023\n\013source_file\030\002 \001" - "(\t\022\r\n\005begin\030\003 \001(\005\022\013\n\003end\030\004 \001(\005B~\n\023com.go" - "ogle.protobufB\020DescriptorProtosH\001Z-googl" - "e.golang.org/protobuf/types/descriptorpb" - "\370\001\001\242\002\003GPB\252\002\032Google.Protobuf.Reflection" + "*\t\010\350\007\020\200\200\200\200\002J\004\010\004\020\005J\004\010\005\020\006J\004\010\006\020\007J\004\010\010\020\tJ\004\010\t\020" + "\n\"\276\003\n\014FieldOptions\022:\n\005ctype\030\001 \001(\0162#.goog" + "le.protobuf.FieldOptions.CType:\006STRING\022\016" + "\n\006packed\030\002 \001(\010\022\?\n\006jstype\030\006 \001(\0162$.google." + "protobuf.FieldOptions.JSType:\tJS_NORMAL\022" + "\023\n\004lazy\030\005 \001(\010:\005false\022\036\n\017unverified_lazy\030" + "\017 \001(\010:\005false\022\031\n\ndeprecated\030\003 \001(\010:\005false\022" + "\023\n\004weak\030\n \001(\010:\005false\022C\n\024uninterpreted_op" + "tion\030\347\007 \003(\0132$.google.protobuf.Uninterpre" + "tedOption\"/\n\005CType\022\n\n\006STRING\020\000\022\010\n\004CORD\020\001" + "\022\020\n\014STRING_PIECE\020\002\"5\n\006JSType\022\r\n\tJS_NORMA" + "L\020\000\022\r\n\tJS_STRING\020\001\022\r\n\tJS_NUMBER\020\002*\t\010\350\007\020\200" + "\200\200\200\002J\004\010\004\020\005\"^\n\014OneofOptions\022C\n\024uninterpre" + "ted_option\030\347\007 \003(\0132$.google.protobuf.Unin" + "terpretedOption*\t\010\350\007\020\200\200\200\200\002\"\223\001\n\013EnumOptio" + "ns\022\023\n\013allow_alias\030\002 \001(\010\022\031\n\ndeprecated\030\003 " + "\001(\010:\005false\022C\n\024uninterpreted_option\030\347\007 \003(" + "\0132$.google.protobuf.UninterpretedOption*" + "\t\010\350\007\020\200\200\200\200\002J\004\010\005\020\006\"}\n\020EnumValueOptions\022\031\n\n" + "deprecated\030\001 \001(\010:\005false\022C\n\024uninterpreted" + "_option\030\347\007 \003(\0132$.google.protobuf.Uninter" + "pretedOption*\t\010\350\007\020\200\200\200\200\002\"{\n\016ServiceOption" + "s\022\031\n\ndeprecated\030! \001(\010:\005false\022C\n\024uninterp" + "reted_option\030\347\007 \003(\0132$.google.protobuf.Un" + "interpretedOption*\t\010\350\007\020\200\200\200\200\002\"\255\002\n\rMethodO" + "ptions\022\031\n\ndeprecated\030! \001(\010:\005false\022_\n\021ide" + "mpotency_level\030\" \001(\0162/.google.protobuf.M" + "ethodOptions.IdempotencyLevel:\023IDEMPOTEN" + "CY_UNKNOWN\022C\n\024uninterpreted_option\030\347\007 \003(" + "\0132$.google.protobuf.UninterpretedOption\"" + "P\n\020IdempotencyLevel\022\027\n\023IDEMPOTENCY_UNKNO" + "WN\020\000\022\023\n\017NO_SIDE_EFFECTS\020\001\022\016\n\nIDEMPOTENT\020" + "\002*\t\010\350\007\020\200\200\200\200\002\"\236\002\n\023UninterpretedOption\022;\n\004" + "name\030\002 \003(\0132-.google.protobuf.Uninterpret" + "edOption.NamePart\022\030\n\020identifier_value\030\003 " + "\001(\t\022\032\n\022positive_int_value\030\004 \001(\004\022\032\n\022negat" + "ive_int_value\030\005 \001(\003\022\024\n\014double_value\030\006 \001(" + "\001\022\024\n\014string_value\030\007 \001(\014\022\027\n\017aggregate_val" + "ue\030\010 \001(\t\0323\n\010NamePart\022\021\n\tname_part\030\001 \002(\t\022" + "\024\n\014is_extension\030\002 \002(\010\"\325\001\n\016SourceCodeInfo" + "\022:\n\010location\030\001 \003(\0132(.google.protobuf.Sou" + "rceCodeInfo.Location\032\206\001\n\010Location\022\020\n\004pat" + "h\030\001 \003(\005B\002\020\001\022\020\n\004span\030\002 \003(\005B\002\020\001\022\030\n\020leading" + "_comments\030\003 \001(\t\022\031\n\021trailing_comments\030\004 \001" + "(\t\022!\n\031leading_detached_comments\030\006 \003(\t\"\247\001" + "\n\021GeneratedCodeInfo\022A\n\nannotation\030\001 \003(\0132" + "-.google.protobuf.GeneratedCodeInfo.Anno" + "tation\032O\n\nAnnotation\022\020\n\004path\030\001 \003(\005B\002\020\001\022\023" + "\n\013source_file\030\002 \001(\t\022\r\n\005begin\030\003 \001(\005\022\013\n\003en" + "d\030\004 \001(\005B~\n\023com.google.protobufB\020Descript" + "orProtosH\001Z-google.golang.org/protobuf/t" + "ypes/descriptorpb\370\001\001\242\002\003GPB\252\002\032Google.Prot" + "obuf.Reflection" ; static ::_pbi::once_flag descriptor_table_google_2fprotobuf_2fdescriptor_2eproto_once; const ::_pbi::DescriptorTable descriptor_table_google_2fprotobuf_2fdescriptor_2eproto = { - false, false, 6078, descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto, + false, false, 6095, descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto, "google/protobuf/descriptor.proto", &descriptor_table_google_2fprotobuf_2fdescriptor_2eproto_once, nullptr, 0, 27, schemas, file_default_instances, TableStruct_google_2fprotobuf_2fdescriptor_2eproto::offsets, @@ -1576,15 +1580,18 @@ class FileDescriptorProto::_Internal { } static const ::PROTOBUF_NAMESPACE_ID::FileOptions& options(const FileDescriptorProto* msg); static void set_has_options(HasBits* has_bits) { - (*has_bits)[0] |= 8u; + (*has_bits)[0] |= 16u; } static const ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo& source_code_info(const FileDescriptorProto* msg); static void set_has_source_code_info(HasBits* has_bits) { - (*has_bits)[0] |= 16u; + (*has_bits)[0] |= 32u; } static void set_has_syntax(HasBits* has_bits) { (*has_bits)[0] |= 4u; } + static void set_has_edition(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } }; const ::PROTOBUF_NAMESPACE_ID::FileOptions& @@ -1617,6 +1624,7 @@ FileDescriptorProto::FileDescriptorProto(const FileDescriptorProto& from) , decltype(_impl_.name_){} , decltype(_impl_.package_){} , decltype(_impl_.syntax_){} + , decltype(_impl_.edition_){} , decltype(_impl_.options_){nullptr} , decltype(_impl_.source_code_info_){nullptr}}; @@ -1645,6 +1653,14 @@ FileDescriptorProto::FileDescriptorProto(const FileDescriptorProto& from) _this->_impl_.syntax_.Set(from._internal_syntax(), _this->GetArenaForAllocation()); } + _impl_.edition_.InitDefault(); + #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING + _impl_.edition_.Set("", GetArenaForAllocation()); + #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING + if (from._internal_has_edition()) { + _this->_impl_.edition_.Set(from._internal_edition(), + _this->GetArenaForAllocation()); + } if (from._internal_has_options()) { _this->_impl_.options_ = new ::PROTOBUF_NAMESPACE_ID::FileOptions(*from._impl_.options_); } @@ -1671,6 +1687,7 @@ inline void FileDescriptorProto::SharedCtor( , decltype(_impl_.name_){} , decltype(_impl_.package_){} , decltype(_impl_.syntax_){} + , decltype(_impl_.edition_){} , decltype(_impl_.options_){nullptr} , decltype(_impl_.source_code_info_){nullptr} }; @@ -1686,6 +1703,10 @@ inline void FileDescriptorProto::SharedCtor( #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.syntax_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING + _impl_.edition_.InitDefault(); + #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING + _impl_.edition_.Set("", GetArenaForAllocation()); + #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } FileDescriptorProto::~FileDescriptorProto() { @@ -1709,6 +1730,7 @@ inline void FileDescriptorProto::SharedDtor() { _impl_.name_.Destroy(); _impl_.package_.Destroy(); _impl_.syntax_.Destroy(); + _impl_.edition_.Destroy(); if (this != internal_default_instance()) delete _impl_.options_; if (this != internal_default_instance()) delete _impl_.source_code_info_; } @@ -1731,7 +1753,7 @@ void FileDescriptorProto::Clear() { _impl_.public_dependency_.Clear(); _impl_.weak_dependency_.Clear(); cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x0000001fu) { + if (cached_has_bits & 0x0000003fu) { if (cached_has_bits & 0x00000001u) { _impl_.name_.ClearNonDefaultToEmpty(); } @@ -1742,10 +1764,13 @@ void FileDescriptorProto::Clear() { _impl_.syntax_.ClearNonDefaultToEmpty(); } if (cached_has_bits & 0x00000008u) { + _impl_.edition_.ClearNonDefaultToEmpty(); + } + if (cached_has_bits & 0x00000010u) { GOOGLE_DCHECK(_impl_.options_ != nullptr); _impl_.options_->Clear(); } - if (cached_has_bits & 0x00000010u) { + if (cached_has_bits & 0x00000020u) { GOOGLE_DCHECK(_impl_.source_code_info_ != nullptr); _impl_.source_code_info_->Clear(); } @@ -1914,6 +1939,18 @@ const char* FileDescriptorProto::_InternalParse(const char* ptr, ::_pbi::ParseCo } else goto handle_unusual; continue; + // optional string edition = 13; + case 13: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 106)) { + auto str = _internal_mutable_edition(); + ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); + CHK_(ptr); + #ifndef NDEBUG + ::_pbi::VerifyUTF8(str, "google.protobuf.FileDescriptorProto.edition"); + #endif // !NDEBUG + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -2008,14 +2045,14 @@ uint8_t* FileDescriptorProto::_InternalSerialize( } // optional .google.protobuf.FileOptions options = 8; - if (cached_has_bits & 0x00000008u) { + if (cached_has_bits & 0x00000010u) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(8, _Internal::options(this), _Internal::options(this).GetCachedSize(), target, stream); } // optional .google.protobuf.SourceCodeInfo source_code_info = 9; - if (cached_has_bits & 0x00000010u) { + if (cached_has_bits & 0x00000020u) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(9, _Internal::source_code_info(this), _Internal::source_code_info(this).GetCachedSize(), target, stream); @@ -2043,6 +2080,16 @@ uint8_t* FileDescriptorProto::_InternalSerialize( 12, this->_internal_syntax(), target); } + // optional string edition = 13; + if (cached_has_bits & 0x00000008u) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::VerifyUTF8StringNamedField( + this->_internal_edition().data(), static_cast(this->_internal_edition().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SERIALIZE, + "google.protobuf.FileDescriptorProto.edition"); + target = stream->WriteStringMaybeAliased( + 13, this->_internal_edition(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); @@ -2114,7 +2161,7 @@ size_t FileDescriptorProto::ByteSizeLong() const { } cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x0000001fu) { + if (cached_has_bits & 0x0000003fu) { // optional string name = 1; if (cached_has_bits & 0x00000001u) { total_size += 1 + @@ -2136,15 +2183,22 @@ size_t FileDescriptorProto::ByteSizeLong() const { this->_internal_syntax()); } - // optional .google.protobuf.FileOptions options = 8; + // optional string edition = 13; if (cached_has_bits & 0x00000008u) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->_internal_edition()); + } + + // optional .google.protobuf.FileOptions options = 8; + if (cached_has_bits & 0x00000010u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.options_); } // optional .google.protobuf.SourceCodeInfo source_code_info = 9; - if (cached_has_bits & 0x00000010u) { + if (cached_has_bits & 0x00000020u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.source_code_info_); @@ -2177,7 +2231,7 @@ void FileDescriptorProto::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, co _this->_impl_.public_dependency_.MergeFrom(from._impl_.public_dependency_); _this->_impl_.weak_dependency_.MergeFrom(from._impl_.weak_dependency_); cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x0000001fu) { + if (cached_has_bits & 0x0000003fu) { if (cached_has_bits & 0x00000001u) { _this->_internal_set_name(from._internal_name()); } @@ -2188,10 +2242,13 @@ void FileDescriptorProto::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, co _this->_internal_set_syntax(from._internal_syntax()); } if (cached_has_bits & 0x00000008u) { + _this->_internal_set_edition(from._internal_edition()); + } + if (cached_has_bits & 0x00000010u) { _this->_internal_mutable_options()->::PROTOBUF_NAMESPACE_ID::FileOptions::MergeFrom( from._internal_options()); } - if (cached_has_bits & 0x00000010u) { + if (cached_has_bits & 0x00000020u) { _this->_internal_mutable_source_code_info()->::PROTOBUF_NAMESPACE_ID::SourceCodeInfo::MergeFrom( from._internal_source_code_info()); } @@ -2246,6 +2303,10 @@ void FileDescriptorProto::InternalSwap(FileDescriptorProto* other) { &_impl_.syntax_, lhs_arena, &other->_impl_.syntax_, rhs_arena ); + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( + &_impl_.edition_, lhs_arena, + &other->_impl_.edition_, rhs_arena + ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(FileDescriptorProto, _impl_.source_code_info_) + sizeof(FileDescriptorProto::_impl_.source_code_info_) diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index 84e40775ea..8a23a88f9e 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -622,6 +622,7 @@ class PROTOBUF_EXPORT FileDescriptorProto final : kNameFieldNumber = 1, kPackageFieldNumber = 2, kSyntaxFieldNumber = 12, + kEditionFieldNumber = 13, kOptionsFieldNumber = 8, kSourceCodeInfoFieldNumber = 9, }; @@ -819,6 +820,24 @@ class PROTOBUF_EXPORT FileDescriptorProto final : std::string* _internal_mutable_syntax(); public: + // optional string edition = 13; + bool has_edition() const; + private: + bool _internal_has_edition() const; + public: + void clear_edition(); + const std::string& edition() const; + template + void set_edition(ArgT0&& arg0, ArgT... args); + std::string* mutable_edition(); + PROTOBUF_NODISCARD std::string* release_edition(); + void set_allocated_edition(std::string* edition); + private: + const std::string& _internal_edition() const; + inline PROTOBUF_ALWAYS_INLINE void _internal_set_edition(const std::string& value); + std::string* _internal_mutable_edition(); + public: + // optional .google.protobuf.FileOptions options = 8; bool has_options() const; private: @@ -875,6 +894,7 @@ class PROTOBUF_EXPORT FileDescriptorProto final : ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr package_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr syntax_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr edition_; ::PROTOBUF_NAMESPACE_ID::FileOptions* options_; ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* source_code_info_; }; @@ -9042,7 +9062,7 @@ FileDescriptorProto::extension() const { // optional .google.protobuf.FileOptions options = 8; inline bool FileDescriptorProto::_internal_has_options() const { - bool value = (_impl_._has_bits_[0] & 0x00000008u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000010u) != 0; PROTOBUF_ASSUME(!value || _impl_.options_ != nullptr); return value; } @@ -9051,7 +9071,7 @@ inline bool FileDescriptorProto::has_options() const { } inline void FileDescriptorProto::clear_options() { if (_impl_.options_ != nullptr) _impl_.options_->Clear(); - _impl_._has_bits_[0] &= ~0x00000008u; + _impl_._has_bits_[0] &= ~0x00000010u; } inline const ::PROTOBUF_NAMESPACE_ID::FileOptions& FileDescriptorProto::_internal_options() const { const ::PROTOBUF_NAMESPACE_ID::FileOptions* p = _impl_.options_; @@ -9069,14 +9089,14 @@ inline void FileDescriptorProto::unsafe_arena_set_allocated_options( } _impl_.options_ = options; if (options) { - _impl_._has_bits_[0] |= 0x00000008u; + _impl_._has_bits_[0] |= 0x00000010u; } else { - _impl_._has_bits_[0] &= ~0x00000008u; + _impl_._has_bits_[0] &= ~0x00000010u; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FileDescriptorProto.options) } inline ::PROTOBUF_NAMESPACE_ID::FileOptions* FileDescriptorProto::release_options() { - _impl_._has_bits_[0] &= ~0x00000008u; + _impl_._has_bits_[0] &= ~0x00000010u; ::PROTOBUF_NAMESPACE_ID::FileOptions* temp = _impl_.options_; _impl_.options_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE @@ -9092,13 +9112,13 @@ inline ::PROTOBUF_NAMESPACE_ID::FileOptions* FileDescriptorProto::release_option } inline ::PROTOBUF_NAMESPACE_ID::FileOptions* FileDescriptorProto::unsafe_arena_release_options() { // @@protoc_insertion_point(field_release:google.protobuf.FileDescriptorProto.options) - _impl_._has_bits_[0] &= ~0x00000008u; + _impl_._has_bits_[0] &= ~0x00000010u; ::PROTOBUF_NAMESPACE_ID::FileOptions* temp = _impl_.options_; _impl_.options_ = nullptr; return temp; } inline ::PROTOBUF_NAMESPACE_ID::FileOptions* FileDescriptorProto::_internal_mutable_options() { - _impl_._has_bits_[0] |= 0x00000008u; + _impl_._has_bits_[0] |= 0x00000010u; if (_impl_.options_ == nullptr) { auto* p = CreateMaybeMessage<::PROTOBUF_NAMESPACE_ID::FileOptions>(GetArenaForAllocation()); _impl_.options_ = p; @@ -9122,9 +9142,9 @@ inline void FileDescriptorProto::set_allocated_options(::PROTOBUF_NAMESPACE_ID:: options = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, options, submessage_arena); } - _impl_._has_bits_[0] |= 0x00000008u; + _impl_._has_bits_[0] |= 0x00000010u; } else { - _impl_._has_bits_[0] &= ~0x00000008u; + _impl_._has_bits_[0] &= ~0x00000010u; } _impl_.options_ = options; // @@protoc_insertion_point(field_set_allocated:google.protobuf.FileDescriptorProto.options) @@ -9132,7 +9152,7 @@ inline void FileDescriptorProto::set_allocated_options(::PROTOBUF_NAMESPACE_ID:: // optional .google.protobuf.SourceCodeInfo source_code_info = 9; inline bool FileDescriptorProto::_internal_has_source_code_info() const { - bool value = (_impl_._has_bits_[0] & 0x00000010u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000020u) != 0; PROTOBUF_ASSUME(!value || _impl_.source_code_info_ != nullptr); return value; } @@ -9141,7 +9161,7 @@ inline bool FileDescriptorProto::has_source_code_info() const { } inline void FileDescriptorProto::clear_source_code_info() { if (_impl_.source_code_info_ != nullptr) _impl_.source_code_info_->Clear(); - _impl_._has_bits_[0] &= ~0x00000010u; + _impl_._has_bits_[0] &= ~0x00000020u; } inline const ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo& FileDescriptorProto::_internal_source_code_info() const { const ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* p = _impl_.source_code_info_; @@ -9159,14 +9179,14 @@ inline void FileDescriptorProto::unsafe_arena_set_allocated_source_code_info( } _impl_.source_code_info_ = source_code_info; if (source_code_info) { - _impl_._has_bits_[0] |= 0x00000010u; + _impl_._has_bits_[0] |= 0x00000020u; } else { - _impl_._has_bits_[0] &= ~0x00000010u; + _impl_._has_bits_[0] &= ~0x00000020u; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:google.protobuf.FileDescriptorProto.source_code_info) } inline ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* FileDescriptorProto::release_source_code_info() { - _impl_._has_bits_[0] &= ~0x00000010u; + _impl_._has_bits_[0] &= ~0x00000020u; ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* temp = _impl_.source_code_info_; _impl_.source_code_info_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE @@ -9182,13 +9202,13 @@ inline ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* FileDescriptorProto::release_sou } inline ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* FileDescriptorProto::unsafe_arena_release_source_code_info() { // @@protoc_insertion_point(field_release:google.protobuf.FileDescriptorProto.source_code_info) - _impl_._has_bits_[0] &= ~0x00000010u; + _impl_._has_bits_[0] &= ~0x00000020u; ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* temp = _impl_.source_code_info_; _impl_.source_code_info_ = nullptr; return temp; } inline ::PROTOBUF_NAMESPACE_ID::SourceCodeInfo* FileDescriptorProto::_internal_mutable_source_code_info() { - _impl_._has_bits_[0] |= 0x00000010u; + _impl_._has_bits_[0] |= 0x00000020u; if (_impl_.source_code_info_ == nullptr) { auto* p = CreateMaybeMessage<::PROTOBUF_NAMESPACE_ID::SourceCodeInfo>(GetArenaForAllocation()); _impl_.source_code_info_ = p; @@ -9212,9 +9232,9 @@ inline void FileDescriptorProto::set_allocated_source_code_info(::PROTOBUF_NAMES source_code_info = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, source_code_info, submessage_arena); } - _impl_._has_bits_[0] |= 0x00000010u; + _impl_._has_bits_[0] |= 0x00000020u; } else { - _impl_._has_bits_[0] &= ~0x00000010u; + _impl_._has_bits_[0] &= ~0x00000020u; } _impl_.source_code_info_ = source_code_info; // @@protoc_insertion_point(field_set_allocated:google.protobuf.FileDescriptorProto.source_code_info) @@ -9286,6 +9306,72 @@ inline void FileDescriptorProto::set_allocated_syntax(std::string* syntax) { // @@protoc_insertion_point(field_set_allocated:google.protobuf.FileDescriptorProto.syntax) } +// optional string edition = 13; +inline bool FileDescriptorProto::_internal_has_edition() const { + bool value = (_impl_._has_bits_[0] & 0x00000008u) != 0; + return value; +} +inline bool FileDescriptorProto::has_edition() const { + return _internal_has_edition(); +} +inline void FileDescriptorProto::clear_edition() { + _impl_.edition_.ClearToEmpty(); + _impl_._has_bits_[0] &= ~0x00000008u; +} +inline const std::string& FileDescriptorProto::edition() const { + // @@protoc_insertion_point(field_get:google.protobuf.FileDescriptorProto.edition) + return _internal_edition(); +} +template +inline PROTOBUF_ALWAYS_INLINE +void FileDescriptorProto::set_edition(ArgT0&& arg0, ArgT... args) { + _impl_._has_bits_[0] |= 0x00000008u; + _impl_.edition_.Set(static_cast(arg0), args..., GetArenaForAllocation()); + // @@protoc_insertion_point(field_set:google.protobuf.FileDescriptorProto.edition) +} +inline std::string* FileDescriptorProto::mutable_edition() { + std::string* _s = _internal_mutable_edition(); + // @@protoc_insertion_point(field_mutable:google.protobuf.FileDescriptorProto.edition) + return _s; +} +inline const std::string& FileDescriptorProto::_internal_edition() const { + return _impl_.edition_.Get(); +} +inline void FileDescriptorProto::_internal_set_edition(const std::string& value) { + _impl_._has_bits_[0] |= 0x00000008u; + _impl_.edition_.Set(value, GetArenaForAllocation()); +} +inline std::string* FileDescriptorProto::_internal_mutable_edition() { + _impl_._has_bits_[0] |= 0x00000008u; + return _impl_.edition_.Mutable(GetArenaForAllocation()); +} +inline std::string* FileDescriptorProto::release_edition() { + // @@protoc_insertion_point(field_release:google.protobuf.FileDescriptorProto.edition) + if (!_internal_has_edition()) { + return nullptr; + } + _impl_._has_bits_[0] &= ~0x00000008u; + auto* p = _impl_.edition_.Release(); +#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING + _impl_.edition_.Set("", GetArenaForAllocation()); +#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING + return p; +} +inline void FileDescriptorProto::set_allocated_edition(std::string* edition) { + if (edition != nullptr) { + _impl_._has_bits_[0] |= 0x00000008u; + } else { + _impl_._has_bits_[0] &= ~0x00000008u; + } + _impl_.edition_.SetAllocated(edition, GetArenaForAllocation()); +#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING + if (_impl_.edition_.IsDefault()) { + _impl_.edition_.Set("", GetArenaForAllocation()); + } +#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING + // @@protoc_insertion_point(field_set_allocated:google.protobuf.FileDescriptorProto.edition) +} + // ------------------------------------------------------------------- // DescriptorProto_ExtensionRange diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index 5b4d06b41c..5404b14190 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -86,8 +86,13 @@ message FileDescriptorProto { optional SourceCodeInfo source_code_info = 9; // The syntax of the proto file. - // The supported values are "proto2" and "proto3". + // The supported values are "proto2", "proto3", and "editions". + // + // If `edition` is present, this value must be "editions". optional string syntax = 12; + + // The edition of the proto file, which is an opaque string. + optional string edition = 13; } // Describes a message type. diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index aefec99081..7d19cf5d52 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -395,7 +395,19 @@ size_t Reflection::SpaceUsedLong(const Message& message) const { } } } +#ifndef PROTOBUF_FUZZ_MESSAGE_SPACE_USED_LONG return total_size; +#else + // Use both `this` and `dummy` to generate the seed so that the scale factor + // is both per-object and non-predictable, but consistent across multiple + // calls in the same binary. + static bool dummy; + uintptr_t seed = + reinterpret_cast(&dummy) ^ reinterpret_cast(this); + // Fuzz the size by +/- 50%. + double scale = (static_cast(seed % 10000) / 10000) + 0.5; + return total_size * scale; +#endif } namespace { diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 82873885bb..d098c81d79 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -298,6 +298,10 @@ class PROTOBUF_EXPORT Message : public MessageLite { // using reflection (rather than the generated code implementation for // ByteSize()). Like ByteSize(), its CPU time is linear in the number of // fields defined for the proto. + // + // Note: The precise value of this method should never be depended on, and can + // change substantially due to internal details. In debug builds, this will + // include a random fuzz factor to prevent these dependencies. virtual size_t SpaceUsedLong() const; PROTOBUF_DEPRECATED_MSG("Please use SpaceUsedLong() instead") diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 1da0410fb9..ba71a9fb45 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -532,6 +532,10 @@ #error PROTOBUF_FORCE_RESET_IN_CLEAR was previously defined #endif +#ifdef PROTOBUF_FUZZ_MESSAGE_SPACE_USED_LONG +#error PROTOBUF_FUZZ_MESSAGE_SPACE_USED_LONG was previously defined +#endif + // Force copy the default string to a string field so that non-optimized builds // have harder-to-rely-on address stability. #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 184980a7e9..4473bdafa4 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -73,6 +73,7 @@ #undef PROTOBUF_FORCE_COPY_IN_SWAP #undef PROTOBUF_FORCE_COPY_IN_MOVE #undef PROTOBUF_FORCE_RESET_IN_CLEAR +#undef PROTOBUF_FUZZ_MESSAGE_SPACE_USED_LONG #undef PROTOBUF_FORCE_COPY_DEFAULT_STRING #undef PROTOBUF_NAMESPACE_OPEN #undef PROTOBUF_NAMESPACE_CLOSE diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 8e9b398360..d452f6e72b 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -503,10 +503,9 @@ int CEscapeInternal(const char* src, int src_len, char* dest, (last_hex_escape && isxdigit(*src)))) { // need space for 4 letter escape and the trailing '\0' to // be written by snprintf. - if (dest_len - used < 5) - return -1; + if (dest_len - used < 5) return -1; snprintf(dest + used, 5, (use_hex ? "\\x%02x" : "\\%03o"), - static_cast(*src)); + static_cast(*src)); is_hex_escape = use_hex; used += 4; } else { diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index c39c10d87b..c15f3f10ae 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -45,6 +45,7 @@ #include #include #include +#include #include // clang-format off @@ -54,35 +55,7 @@ namespace google { namespace protobuf { namespace util { - -namespace internal { -ZeroCopyStreamByteSink::~ZeroCopyStreamByteSink() { - if (buffer_size_ > 0) { - stream_->BackUp(buffer_size_); - } -} - -void ZeroCopyStreamByteSink::Append(const char* bytes, size_t len) { - while (true) { - if (len <= buffer_size_) { // NOLINT - memcpy(buffer_, bytes, len); - buffer_ = static_cast(buffer_) + len; - buffer_size_ -= len; - return; - } - if (buffer_size_ > 0) { - memcpy(buffer_, bytes, buffer_size_); - bytes += buffer_size_; - len -= buffer_size_; - } - if (!stream_->Next(&buffer_, &buffer_size_)) { - // There isn't a way for ByteSink to report errors. - buffer_size_ = 0; - return; - } - } -} -} // namespace internal +using ::google::protobuf::util::zc_sink_internal::ZeroCopyStreamByteSink; util::Status BinaryToJsonStream(TypeResolver* resolver, const std::string& type_url, @@ -181,7 +154,7 @@ util::Status JsonToBinaryStream(TypeResolver* resolver, const JsonParseOptions& options) { google::protobuf::Type type; RETURN_IF_ERROR(resolver->ResolveMessageType(type_url, &type)); - internal::ZeroCopyStreamByteSink sink(binary_output); + ZeroCopyStreamByteSink sink(binary_output); StatusErrorListener listener; converter::ProtoStreamObjectWriter::Options proto_writer_options; proto_writer_options.ignore_unknown_fields = options.ignore_unknown_fields; diff --git a/src/google/protobuf/util/json_util.h b/src/google/protobuf/util/json_util.h index 0f1c4d8b5f..4f4594e2f9 100644 --- a/src/google/protobuf/util/json_util.h +++ b/src/google/protobuf/util/json_util.h @@ -175,26 +175,6 @@ inline util::Status JsonToBinaryString(TypeResolver* resolver, return JsonToBinaryString(resolver, type_url, json_input, binary_output, JsonParseOptions()); } - -namespace internal { -// Internal helper class. Put in the header so we can write unit-tests for it. -class PROTOBUF_EXPORT ZeroCopyStreamByteSink : public strings::ByteSink { - public: - explicit ZeroCopyStreamByteSink(io::ZeroCopyOutputStream* stream) - : stream_(stream), buffer_(nullptr), buffer_size_(0) {} - ~ZeroCopyStreamByteSink() override; - - void Append(const char* bytes, size_t len) override; - - private: - io::ZeroCopyOutputStream* stream_; - void* buffer_; - int buffer_size_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ZeroCopyStreamByteSink); -}; -} // namespace internal - } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index 29ffe865d9..10494e55a7 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -632,139 +632,6 @@ TEST(JsonUtilTest, TestParsingEnumIgnoreCase) { ASSERT_EQ(m.enum_value(), proto3::BAR); } -// A ZeroCopyOutputStream that writes to multiple buffers. -class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream { - public: - explicit SegmentedZeroCopyOutputStream( - std::vector segments) - : segments_(segments) { - // absl::c_* functions are not cloned in OSS. - std::reverse(segments_.begin(), segments_.end()); - } - - bool Next(void** buffer, int* length) override { - if (segments_.empty()) { - return false; - } - last_segment_ = segments_.back(); - segments_.pop_back(); - // TODO(b/234159981): This is only ever constructed in test code, and only - // from non-const bytes, so this is a valid cast. We need to do this since - // OSS proto does not yet have absl::Span; once we take a full Abseil - // dependency we should use that here instead. - *buffer = const_cast(last_segment_.data()); - *length = static_cast(last_segment_.size()); - byte_count_ += static_cast(last_segment_.size()); - return true; - } - - void BackUp(int length) override { - GOOGLE_CHECK(length <= static_cast(last_segment_.size())); - - size_t backup = last_segment_.size() - static_cast(length); - segments_.push_back(last_segment_.substr(backup)); - last_segment_ = last_segment_.substr(0, backup); - byte_count_ -= static_cast(length); - } - - int64_t ByteCount() const override { return byte_count_; } - - private: - std::vector segments_; - StringPiece last_segment_; - int64_t byte_count_ = 0; -}; - -// This test splits the output buffer and also the input data into multiple -// segments and checks that the implementation of ZeroCopyStreamByteSink -// handles all possible cases correctly. -TEST(ZeroCopyStreamByteSinkTest, TestAllInputOutputPatterns) { - static constexpr int kOutputBufferLength = 10; - // An exhaustive test takes too long, skip some combinations to make the test - // run faster. - static constexpr int kSkippedPatternCount = 7; - - char buffer[kOutputBufferLength]; - for (int split_pattern = 0; split_pattern < (1 << (kOutputBufferLength - 1)); - split_pattern += kSkippedPatternCount) { - // Split the buffer into small segments according to the split_pattern. - std::vector segments; - int segment_start = 0; - for (int i = 0; i < kOutputBufferLength - 1; ++i) { - if (split_pattern & (1 << i)) { - segments.push_back( - StringPiece(buffer + segment_start, i - segment_start + 1)); - segment_start = i + 1; - } - } - segments.push_back(StringPiece(buffer + segment_start, - kOutputBufferLength - segment_start)); - - // Write exactly 10 bytes through the ByteSink. - std::string input_data = "0123456789"; - for (int input_pattern = 0; input_pattern < (1 << (input_data.size() - 1)); - input_pattern += kSkippedPatternCount) { - memset(buffer, 0, sizeof(buffer)); - { - SegmentedZeroCopyOutputStream output_stream(segments); - internal::ZeroCopyStreamByteSink byte_sink(&output_stream); - int start = 0; - for (int j = 0; j < input_data.length() - 1; ++j) { - if (input_pattern & (1 << j)) { - byte_sink.Append(&input_data[start], j - start + 1); - start = j + 1; - } - } - byte_sink.Append(&input_data[start], input_data.length() - start); - } - EXPECT_EQ(std::string(buffer, input_data.length()), input_data); - } - - // Write only 9 bytes through the ByteSink. - input_data = "012345678"; - for (int input_pattern = 0; input_pattern < (1 << (input_data.size() - 1)); - input_pattern += kSkippedPatternCount) { - memset(buffer, 0, sizeof(buffer)); - { - SegmentedZeroCopyOutputStream output_stream(segments); - internal::ZeroCopyStreamByteSink byte_sink(&output_stream); - int start = 0; - for (int j = 0; j < input_data.length() - 1; ++j) { - if (input_pattern & (1 << j)) { - byte_sink.Append(&input_data[start], j - start + 1); - start = j + 1; - } - } - byte_sink.Append(&input_data[start], input_data.length() - start); - } - EXPECT_EQ(std::string(buffer, input_data.length()), input_data); - EXPECT_EQ(buffer[input_data.length()], 0); - } - - // Write 11 bytes through the ByteSink. The extra byte will just - // be ignored. - input_data = "0123456789A"; - for (int input_pattern = 0; input_pattern < (1 << (input_data.size() - 1)); - input_pattern += kSkippedPatternCount) { - memset(buffer, 0, sizeof(buffer)); - { - SegmentedZeroCopyOutputStream output_stream(segments); - internal::ZeroCopyStreamByteSink byte_sink(&output_stream); - int start = 0; - for (int j = 0; j < input_data.length() - 1; ++j) { - if (input_pattern & (1 << j)) { - byte_sink.Append(&input_data[start], j - start + 1); - start = j + 1; - } - } - byte_sink.Append(&input_data[start], input_data.length() - start); - } - EXPECT_EQ(input_data.substr(0, kOutputBufferLength), - std::string(buffer, kOutputBufferLength)); - } - } -} - TEST(JsonUtilTest, TestWrongJsonInput) { StringPiece json = "{\"unknown_field\":\"some_value\"}"; io::ArrayInputStream input_stream(json.data(), json.size()); diff --git a/src/google/protobuf/util/zero_copy_sink.cc b/src/google/protobuf/util/zero_copy_sink.cc new file mode 100644 index 0000000000..2b24f11238 --- /dev/null +++ b/src/google/protobuf/util/zero_copy_sink.cc @@ -0,0 +1,60 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +namespace google { +namespace protobuf { +namespace util { +namespace zc_sink_internal { +void ZeroCopyStreamByteSink::Append(const char* bytes, size_t len) { + while (true) { + if (len <= buffer_size_) { // NOLINT + memcpy(buffer_, bytes, len); + buffer_ = static_cast(buffer_) + len; + buffer_size_ -= len; + return; + } + if (buffer_size_ > 0) { + memcpy(buffer_, bytes, buffer_size_); + bytes += buffer_size_; + len -= buffer_size_; + } + if (!stream_->Next(&buffer_, &buffer_size_)) { + // There isn't a way for ByteSink to report errors. + buffer_size_ = 0; + return; + } + } +} +} // namespace zc_sink_internal +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/zero_copy_sink.h b/src/google/protobuf/util/zero_copy_sink.h new file mode 100644 index 0000000000..f2fd9d7867 --- /dev/null +++ b/src/google/protobuf/util/zero_copy_sink.h @@ -0,0 +1,74 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_UTIL_ZERO_COPY_SINK_H__ +#define GOOGLE_PROTOBUF_UTIL_ZERO_COPY_SINK_H__ + +#include + +#include +#include +#include + +// Must be included last. +#include + +namespace google { +namespace protobuf { +namespace util { +namespace zc_sink_internal { +// Internal helper class. Put in the header so we can write unit-tests for it. +class PROTOBUF_EXPORT ZeroCopyStreamByteSink : public strings::ByteSink { + public: + explicit ZeroCopyStreamByteSink(io::ZeroCopyOutputStream* stream) + : stream_(stream) {} + + ~ZeroCopyStreamByteSink() override { + if (buffer_size_ > 0) { + stream_->BackUp(buffer_size_); + } + } + + void Append(const char* bytes, size_t len) override; + + private: + io::ZeroCopyOutputStream* stream_; + void* buffer_ = nullptr; + int buffer_size_ = 0; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ZeroCopyStreamByteSink); +}; +} // namespace zc_sink_internal +} // namespace util +} // namespace protobuf +} // namespace google + +#include +#endif // GOOGLE_PROTOBUF_UTIL_ZERO_COPY_SINK_H__ diff --git a/src/google/protobuf/util/zero_copy_sink_test.cc b/src/google/protobuf/util/zero_copy_sink_test.cc new file mode 100644 index 0000000000..9eed0fe160 --- /dev/null +++ b/src/google/protobuf/util/zero_copy_sink_test.cc @@ -0,0 +1,226 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace util { +namespace zc_sink_internal { +namespace { + +class ChunkedString { + public: + explicit ChunkedString(StringPiece data, size_t skipped_patterns) + : data_(data), skipped_patterns_(skipped_patterns) {} + + // Returns the next chunk; empty if out of chunks. + StringPiece NextChunk() { + if (pattern_bit_idx_ == data_.size()) { + return ""; + } + size_t start = pattern_bit_idx_; + do { + ++pattern_bit_idx_; + } while (pattern_bit_idx_ < data_.size() && + (pattern_ >> pattern_bit_idx_ & 1) == 0); + size_t end = pattern_bit_idx_; + return data_.substr(start, end - start); + } + + // Resets the stream and runs the next pattern of splits. + bool NextPattern() { + pattern_ += skipped_patterns_; + if (pattern_ >= (1 << data_.size())) { + return false; + } + pattern_bit_idx_ = 0; + return true; + } + + // prints out the pattern as a sequence of quoted strings. + std::string PatternAsQuotedString() { + std::string out; + size_t start = 0; + for (size_t i = 0; i <= data_.size(); ++i) { + if (i == data_.size() || (pattern_ >> start & 1) != 0) { + if (!out.empty()) { + StrAppend(&out, " "); + } + StrAppend( + &out, "\"", + strings::CHexEscape(std::string{data_.substr(start, i - start)}), + "\""); + start = i; + } + } + return out; + } + + private: + StringPiece data_; + size_t skipped_patterns_; + // pattern_ is a bitset indicating at which indices we insert a seam. + uint64_t pattern_ = 0; + size_t pattern_bit_idx_ = 0; +}; + +class PatternedOutputStream : public io::ZeroCopyOutputStream { + public: + explicit PatternedOutputStream(ChunkedString data) : data_(data) {} + + bool Next(void** buffer, int* length) override { + StringPiece segment; + if (!back_up_.empty()) { + segment = back_up_.back(); + back_up_.pop_back(); + } else { + segment_ = data_.NextChunk(); + segment = segment_; + } + + if (segment_.empty()) { + return false; + } + + // TODO(b/234159981): This is only ever constructed in test code, and only + // from non-const bytes, so this is a valid cast. We need to do this since + // OSS proto does not yet have absl::Span; once we take a full Abseil + // dependency we should use that here instead. + *buffer = const_cast(segment.data()); + *length = static_cast(segment.size()); + byte_count_ += static_cast(segment.size()); + return true; + } + + void BackUp(int length) override { + GOOGLE_CHECK(length <= static_cast(segment_.size())); + + size_t backup = segment_.size() - static_cast(length); + back_up_.push_back(segment_.substr(backup)); + segment_ = segment_.substr(0, backup); + byte_count_ -= static_cast(length); + } + + int64_t ByteCount() const override { return byte_count_; } + + private: + ChunkedString data_; + StringPiece segment_; + + std::vector back_up_; + int64_t byte_count_ = 0; +}; + +class ZeroCopyStreamByteSinkTest : public testing::Test { + protected: + std::array output_{}; + StringPiece output_view_{output_.data(), output_.size()}; + ChunkedString output_chunks_{output_view_, 7}; +}; + +TEST_F(ZeroCopyStreamByteSinkTest, WriteExact) { + do { + SCOPED_TRACE(output_chunks_.PatternAsQuotedString()); + ChunkedString input("0123456789", 1); + + do { + SCOPED_TRACE(input.PatternAsQuotedString()); + output_ = {}; + PatternedOutputStream output_stream(output_chunks_); + ZeroCopyStreamByteSink byte_sink(&output_stream); + SCOPED_TRACE(input.PatternAsQuotedString()); + StringPiece chunk; + while (!(chunk = input.NextChunk()).empty()) { + byte_sink.Append(chunk.data(), chunk.size()); + } + } while (input.NextPattern()); + + ASSERT_EQ(output_view_, "0123456789"); + } while (output_chunks_.NextPattern()); +} + +TEST_F(ZeroCopyStreamByteSinkTest, WriteShort) { + do { + SCOPED_TRACE(output_chunks_.PatternAsQuotedString()); + ChunkedString input("012345678", 1); + + do { + SCOPED_TRACE(input.PatternAsQuotedString()); + output_ = {}; + PatternedOutputStream output_stream(output_chunks_); + ZeroCopyStreamByteSink byte_sink(&output_stream); + SCOPED_TRACE(input.PatternAsQuotedString()); + StringPiece chunk; + while (!(chunk = input.NextChunk()).empty()) { + byte_sink.Append(chunk.data(), chunk.size()); + } + } while (input.NextPattern()); + + ASSERT_EQ(output_view_, StringPiece("012345678\0", 10)); + } while (output_chunks_.NextPattern()); +} + +TEST_F(ZeroCopyStreamByteSinkTest, WriteLong) { + do { + SCOPED_TRACE(output_chunks_.PatternAsQuotedString()); + ChunkedString input("0123456789A", 1); + + do { + SCOPED_TRACE(input.PatternAsQuotedString()); + output_ = {}; + PatternedOutputStream output_stream(output_chunks_); + ZeroCopyStreamByteSink byte_sink(&output_stream); + SCOPED_TRACE(input.PatternAsQuotedString()); + StringPiece chunk; + while (!(chunk = input.NextChunk()).empty()) { + byte_sink.Append(chunk.data(), chunk.size()); + } + } while (input.NextPattern()); + + ASSERT_EQ(output_view_, "0123456789"); + } while (output_chunks_.NextPattern()); +} +} // namespace +} // namespace zc_sink_internal +} // namespace util +} // namespace protobuf +} // namespace google