From e9927cf762596bebeef95be9af8ab7aef9a703d4 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 17 Jul 2024 14:02:52 -0700 Subject: [PATCH 001/107] Replace matrix.version with cache_key PiperOrigin-RevId: 653354465 --- .github/workflows/test_cpp.yml | 10 +++++----- .github/workflows/test_java.yml | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 854f93484c..4b36b561c2 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -31,19 +31,19 @@ jobs: # Override cases with custom images - config: { name: "Bazel7", flags: --noenable_bzlmod } - version: Bazel7 + cache_key: Bazel7 image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e" targets: "//src/... //third_party/utf8_range/..." - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod --enable_workspace } - version: Bazel7bzlmod + cache_key: Bazel7bzlmod image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e" targets: "//src/... //third_party/utf8_range/..." - config: { name: "TCMalloc" } - version: TcMalloc + cache_key: TcMalloc image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:1c5133455481f4d1bb8afa477029604f41f1a3c46cebe4d9958cf1af95b5c87c" targets: "//src/... //third_party/utf8_range/..." - config: { name: "aarch64" } - version: TcMalloc + cache_key: TcMalloc targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test //third_party/utf8_range/..." image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:6.3.0-aarch64-68e662b3a56b881804dc4e9d45f949791cbc4b94" name: Linux ${{ matrix.config.name }} @@ -58,7 +58,7 @@ jobs: with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel-cache: cpp_linux/${{ matrix.version }} + bazel-cache: cpp_linux/${{ matrix.cache_key }} bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} exclude-targets: ${{ matrix.exclude-targets }} diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 64639b9d31..189e5c2907 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -18,31 +18,31 @@ jobs: matrix: include: - name: OpenJDK 8 - version: '8' + cache_key: '8' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 # TODO: b/318555165 - enable the layering check. Currently it does # not work correctly with the toolchain in this Docker image. targets: //java/... //java/internal:java_version //compatibility/... --features=-layering_check - name: OpenJDK 11 - version: '11' + cache_key: '11' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 targets: //java/... //java/internal:java_version //compatibility/... - name: OpenJDK 17 - version: '17' + cache_key: '17' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 targets: //java/... //java/internal:java_version //compatibility/... - name: Bazel7 - version: 'bazel7nobzlmod' + cache_key: 'bazel7nobzlmod' image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e targets: //java/... //java/internal:java_version //compatibility/... flags: --noenable_bzlmod - name: Bazel7 with Bzlmod - version: 'bazel7bzlmod' + cache_key: 'bazel7bzlmod' image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e targets: //java/... //java/internal:java_version //compatibility/... flags: --enable_bzlmod --enable_workspace - name: aarch64 - version: 'aarch64' + cache_key: 'aarch64' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17 targets: //java/... //compatibility/... //src/google/protobuf/compiler:protoc_aarch64_test @@ -58,7 +58,7 @@ jobs: with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel-cache: java_linux/${{ matrix.version }} + bazel-cache: java_linux/${{ matrix.cache_key }} bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_JAVA_VERSION # TODO restore this test (or a better one) when gRPC has rebuilt with 26.x From 4b230be0d004cb4b6c69bcc8ff994cbe949b1475 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Wed, 17 Jul 2024 15:21:49 -0700 Subject: [PATCH 002/107] Implement the lazy repeated field data structure. PiperOrigin-RevId: 653379333 --- src/google/protobuf/lazy_repeated_field.cc | 344 +++++ src/google/protobuf/lazy_repeated_field.h | 1123 +++++++++++++++++ .../protobuf/lazy_repeated_field_heavy.cc | 401 ++++++ src/google/protobuf/repeated_ptr_field.h | 1 + 4 files changed, 1869 insertions(+) create mode 100644 src/google/protobuf/lazy_repeated_field.cc create mode 100644 src/google/protobuf/lazy_repeated_field.h create mode 100644 src/google/protobuf/lazy_repeated_field_heavy.cc diff --git a/src/google/protobuf/lazy_repeated_field.cc b/src/google/protobuf/lazy_repeated_field.cc new file mode 100644 index 0000000000..e3b83ad04d --- /dev/null +++ b/src/google/protobuf/lazy_repeated_field.cc @@ -0,0 +1,344 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "google/protobuf/lazy_repeated_field.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/log/log.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/generated_message_util.h" +#include "google/protobuf/io/coded_stream.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/message_lite.h" +#include "google/protobuf/parse_context.h" + +// Must be included last. +// clang-format off +#include "google/protobuf/port_def.inc" +#include "google/protobuf/repeated_ptr_field.h" +// clang-format on + +namespace google { +namespace protobuf { +namespace internal { +namespace {} // namespace + +namespace { + +inline const char* InternalParseRepeated(const char* ptr, + ParseContext* local_ctx, + RepeatedPtrFieldBase* value, + const MessageLite* prototype) { + uint32_t expected_tag; + ptr = ReadTag(ptr, &expected_tag); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + // TODO: Try to optimize this. The tags and lengths are read again + // which is a bit wasteful. + return LazyRepeatedPtrField::ParseToRepeatedMessage( + ptr, local_ctx, prototype, expected_tag, value); +} + +template +inline bool ParseWithNullOuterContextImpl(const T& input, + RepeatedPtrFieldBase* value, + const MessageLite* prototype, + bool set_missing_required) { + // Null outer context means it's either already verified or unverified. + // + // If the payload is eagerly verified, the recursion limit was also verified + // and we don't need to repeat that. Also, users might have used a custom + // limit which is not known at this access. + // + // Unverified lazy fields may suffer from stack overflow with deeply nested + // data. We argue that it should be better than silent data corruption. + constexpr int kUnlimitedDepth = std::numeric_limits::max(); + const char* ptr; + ParseContext local_ctx(kUnlimitedDepth, false, &ptr, input); + + if (set_missing_required) { + local_ctx.SetParentMissingRequiredFields(); + } + // Unparsed data is already verified at parsing. Disable eager-verification. + (void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy); + + ptr = InternalParseRepeated(ptr, &local_ctx, value, prototype); + return ptr != nullptr && + (local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit()); +} + +template +inline bool ParseWithOuterContextImpl(const T& input, ParseContext* ctx, + RepeatedPtrFieldBase* value, + const MessageLite* prototype, + bool set_missing_required) { + if (ctx == nullptr) { + return ParseWithNullOuterContextImpl(input, value, prototype, + set_missing_required); + } + + ABSL_DCHECK(!ctx->AliasingEnabled()); + // set_missing_required => ctx == nullptr + ABSL_DCHECK(!set_missing_required); + + // Create local context with depth. + const char* ptr; + ParseContext local_ctx(ParseContext::kSpawn, *ctx, &ptr, input); + + if (set_missing_required) { + local_ctx.SetParentMissingRequiredFields(); + } + if (ctx->lazy_parse_mode() == ParseContext::LazyParseMode::kEagerVerify) { + // Unparsed data is already verified at parsing. Disable eager-verification. + (void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy); + } + + ptr = InternalParseRepeated(ptr, &local_ctx, value, prototype); + + if (local_ctx.missing_required_fields()) { + ctx->SetMissingRequiredFields(); + } + + return ptr != nullptr && + (local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit()); +} + +class ByPrototype { + public: + explicit ByPrototype(const MessageLite* prototype) : prototype_(prototype) {} + + MessageLite* New(Arena* arena) const { return prototype_->New(arena); } + + const MessageLite& Default() const { return *prototype_; } + + private: + const MessageLite* prototype_; +}; +} // namespace + +const RepeatedPtrFieldBase* LazyRepeatedPtrField::GetByPrototype( + const MessageLite* prototype, Arena* arena, ParseContext* ctx) const { + return GetGeneric(ByPrototype(prototype), arena, ctx); +} + +RepeatedPtrFieldBase* LazyRepeatedPtrField::MutableByPrototype( + const MessageLite* prototype, Arena* arena, ParseContext* ctx) { + return MutableGeneric(ByPrototype(prototype), arena, ctx); +} + +void LazyRepeatedPtrField::Clear() { + PerformTransition([](ExclusiveTxn& txn) { + auto* value = txn.mutable_value(); + if (value != nullptr) value->Clear>(); + return RawState::kCleared; + }); +} + +bool LazyRepeatedPtrField::IsEagerSerializeSafe(const MessageLite* prototype, + int32_t number, + Arena* arena) const { + // "prototype" may be null if it is for dynamic messages. This is ok as + // dynamic extensions won't be lazy as they lack verify functions any way. + if (prototype == nullptr) return false; + + for (;;) { + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + case LogicalState::kDirty: + return true; + case LogicalState::kNoParseRequired: { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + size_t tag_size = WireFormatLite::TagSize( + number, WireFormatLite::FieldType::TYPE_MESSAGE); + size_t total_size = tag_size * value->size(); + for (int i = 0; i < value->size(); i++) { + total_size += WireFormatLite::LengthDelimitedSize( + value->Get>(i).ByteSizeLong()); + } + return total_size == unparsed_.Size(); + } + case LogicalState::kParseRequired: { + GetByPrototype(prototype, arena); + break; // reswitch + } + } + } + // Required for certain compiler configurations. + ABSL_LOG(FATAL) << "Not reachable"; + return false; +} + +void LazyRepeatedPtrField::swap_atomics(std::atomic& lhs, + std::atomic& rhs) { + auto l = lhs.exchange(rhs.load(std::memory_order_relaxed), + std::memory_order_relaxed); + rhs.store(l, std::memory_order_relaxed); +} + +void LazyRepeatedPtrField::Swap(LazyRepeatedPtrField* lhs, Arena* lhs_arena, + LazyRepeatedPtrField* rhs, Arena* rhs_arena) { + static auto reallocate = [](LazyRepeatedPtrField* f, Arena* arena, + bool cleanup_old) { + auto raw = f->raw_.load(std::memory_order_relaxed); + if (raw.value() != nullptr) { + auto* new_value = Arena::Create(arena); + if (!raw.value()->empty()) { + new_value->MergeFrom(*raw.value()); + } + if (cleanup_old) { + delete reinterpret_cast*>( + raw.value()); + }; + raw.set_value(new_value); + f->raw_.store(raw, std::memory_order_relaxed); + } + auto old_unparsed = f->unparsed_; + f->unparsed_.Visit( + [] {}, + [&](auto& cord) { f->unparsed_.InitAsCord(arena, std::move(cord)); }, + [&](auto view) { + if (arena == nullptr) { + f->unparsed_.InitAsCord(arena, view); + } else { + f->unparsed_.InitAndSetArray(arena, view); + } + }); + if (cleanup_old) old_unparsed.Destroy(); + }; + static auto take_ownership = [](LazyRepeatedPtrField* f, Arena* arena) { +#ifdef PROTOBUF_FORCE_COPY_IN_SWAP + reallocate(f, arena, true); +#else + arena->Own(reinterpret_cast*>( + f->raw_.load(std::memory_order_relaxed).mutable_value())); + f->unparsed_.TransferHeapOwnershipToArena(arena); +#endif + }; + + using std::swap; // Enable ADL with fallback + swap_atomics(lhs->raw_, rhs->raw_); + swap(lhs->unparsed_, rhs->unparsed_); + // At this point we are in a weird state. The messages have been swapped into + // their destination, but we have completely ignored the arenas, so the owning + // arena is actually on the opposite message. Now we straighten out our + // ownership by forcing reallocations/ownership changes as needed. + if (lhs_arena == rhs_arena) { +#ifdef PROTOBUF_FORCE_COPY_IN_SWAP + if (lhs_arena == nullptr) { + reallocate(lhs, lhs_arena, true); + reallocate(rhs, rhs_arena, true); + } +#endif + } else { + if (lhs_arena == nullptr) { + take_ownership(rhs, rhs_arena); + reallocate(lhs, lhs_arena, false); + } else if (rhs_arena == nullptr) { + take_ownership(lhs, lhs_arena); + reallocate(rhs, rhs_arena, false); + } else { + reallocate(lhs, lhs_arena, false); + reallocate(rhs, rhs_arena, false); + } + } +} + +void LazyRepeatedPtrField::InternalSwap( + LazyRepeatedPtrField* PROTOBUF_RESTRICT lhs, + LazyRepeatedPtrField* PROTOBUF_RESTRICT rhs) { + using std::swap; // Enable ADL with fallback + swap_atomics(lhs->raw_, rhs->raw_); + swap(lhs->unparsed_, rhs->unparsed_); +} + +bool LazyRepeatedPtrField::ParseWithOuterContext(RepeatedPtrFieldBase* value, + const absl::Cord& input, + ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required) { + absl::optional flat = input.TryFlat(); + if (flat.has_value()) { + return ParseWithOuterContextImpl(*flat, ctx, value, prototype, + set_missing_required); + } + + io::CordInputStream cis(&input); + return ParseWithOuterContextImpl(&cis, ctx, value, prototype, + set_missing_required); +} + +bool LazyRepeatedPtrField::ParseWithOuterContext(RepeatedPtrFieldBase* value, + absl::string_view input, + ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required) { + return ParseWithOuterContextImpl(input, ctx, value, prototype, + set_missing_required); +} + +size_t LazyRepeatedPtrField::ByteSizeLong(size_t tag_size) const { + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: + return unparsed_.Size(); + + case LogicalState::kDirty: + const auto* value = raw_.load(std::memory_order_relaxed).value(); + size_t total_size = tag_size * value->size(); + for (int i = 0; i < value->size(); i++) { + total_size += WireFormatLite::LengthDelimitedSize( + value->Get>(i).ByteSizeLong()); + } + return total_size; + } + // Required for certain compiler configurations. + ABSL_LOG(FATAL) << "Not reachable"; + return -1; +} + +void LazyRepeatedPtrField::LogParseError(const RepeatedPtrFieldBase* value) { + const MessageLite* message = + &value->at>(0); + auto get_error_string = [&value]() { + std::string str; + for (int i = 0; i < value->size(); i++) { + absl::StrAppend(&str, "[", i, "]: ", + value->at>(i) + .InitializationErrorString(), + "\n"); + } + return str; + }; +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) + // In fuzzing mode, we log less to speed up fuzzing. + ABSL_LOG_EVERY_N(INFO, 100000) +#else + ABSL_LOG_EVERY_N_SEC(INFO, 1) +#endif + << "Lazy parsing failed for RepeatedPtrField<" << message->GetTypeName() + << "> error=" << get_error_string() << " (N = " << COUNTER << ")"; +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" diff --git a/src/google/protobuf/lazy_repeated_field.h b/src/google/protobuf/lazy_repeated_field.h new file mode 100644 index 0000000000..ab573e58c0 --- /dev/null +++ b/src/google/protobuf/lazy_repeated_field.h @@ -0,0 +1,1123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_LAZY_REPEATED_FIELD_H__ +#define GOOGLE_PROTOBUF_LAZY_REPEATED_FIELD_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/log/absl_check.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/generated_message_util.h" +#include "google/protobuf/internal_visibility.h" +#include "google/protobuf/io/coded_stream.h" +#include "google/protobuf/message_lite.h" +#include "google/protobuf/parse_context.h" +#include "google/protobuf/port.h" +#include "google/protobuf/raw_ptr.h" +#include "google/protobuf/repeated_ptr_field.h" +#include "google/protobuf/wire_format_verify.h" + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +// must be last +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { + +class Descriptor; +namespace io { +class CodedInputStream; +class CodedOutputStream; +} // namespace io +} // namespace protobuf +} // namespace google + +namespace google { +namespace protobuf { +namespace internal { + +inline const char* ReadTagInternal(const char* ptr, uint8_t* tag) { + *tag = UnalignedLoad(ptr); + return ptr + sizeof(uint8_t); +} + +inline const char* ReadTagInternal(const char* ptr, uint16_t* tag) { + *tag = UnalignedLoad(ptr); + return ptr + sizeof(uint16_t); +} + +inline const char* ReadTagInternal(const char* ptr, uint32_t* tag) { + return ReadTag(ptr, tag); +} + +template +inline size_t TagSizeInternal(TagType tag); +template <> +inline size_t TagSizeInternal(uint8_t tag) { + return sizeof(uint8_t); +} +template <> +inline size_t TagSizeInternal(uint16_t tag) { + return sizeof(uint16_t); +} +template <> +inline size_t TagSizeInternal(uint32_t tag) { + return io::CodedOutputStream::VarintSize32(tag); +} + +// This class is used to represent lazily-loaded repeated message fields. +// It stores the field in a raw buffer or a Cord initially, and then parses that +// on-demand if a caller asks for the RepeatedPtrField object. +// +// As with most protobuf classes, const methods of this class are safe to call +// from multiple threads at once, but non-const methods may only be called when +// the thread has guaranteed that it has exclusive access to the field. +class LazyRepeatedPtrField { + public: + constexpr LazyRepeatedPtrField() : raw_(MessageState(RawState::kCleared)) {} + LazyRepeatedPtrField(const LazyRepeatedPtrField& rhs) + : LazyRepeatedPtrField(nullptr, rhs, nullptr) {} + + // Arena enabled constructors. + LazyRepeatedPtrField(internal::InternalVisibility, Arena* arena) + : LazyRepeatedPtrField(arena) {} + LazyRepeatedPtrField(internal::InternalVisibility, Arena* arena, + const LazyRepeatedPtrField& rhs, Arena* rhs_arena) + : LazyRepeatedPtrField(arena, rhs, rhs_arena) {} + + // TODO: make this constructor private + explicit constexpr LazyRepeatedPtrField(Arena*) + : raw_(MessageState(RawState::kCleared)) {} + + LazyRepeatedPtrField& operator=(const LazyRepeatedPtrField&) = delete; + + ~LazyRepeatedPtrField(); + + bool IsClear() const { + auto state = GetLogicalState(); + return state == LogicalState::kClear || + state == LogicalState::kClearExposed; + } + + // Get and Mutable trigger parsing. + template + const RepeatedPtrField& Get(const Element* default_instance, + Arena* arena) const { + return *reinterpret_cast*>( + GetGeneric(ByTemplate(default_instance), arena, nullptr)); + } + + template + RepeatedPtrField* Mutable(const Element* default_instance, + Arena* arena) { + return reinterpret_cast*>( + MutableGeneric(ByTemplate(default_instance), arena, nullptr)); + } + + bool IsInitialized(const MessageLite* prototype, Arena* arena) const { + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: { + return true; + } + case LogicalState::kParseRequired: + case LogicalState::kNoParseRequired: { + // Returns true if "unparsed" is not verified to be (maybe) + // uninitialized. Otherwise, falls through to next cases to eagerly + // parse message and call IsInitialized(). + if (!MaybeUninitialized()) return true; + } + ABSL_FALLTHROUGH_INTENDED; + case LogicalState::kDirty: { + const auto& value = *GetByPrototype(prototype, arena); + for (int i = 0; i < value.size(); ++i) { + if (!value.Get>(i).IsInitialized()) + return false; + } + return true; + } + default: + __builtin_unreachable(); + } + } + + // Dynamic versions of basic accessors. + const RepeatedPtrFieldBase* GetDynamic(const Descriptor* type, + MessageFactory* factory, + Arena* arena) const; + RepeatedPtrFieldBase* MutableDynamic(const Descriptor* type, + MessageFactory* factory, Arena* arena); + + // Basic accessors that use a default instance to create the message. + const RepeatedPtrFieldBase* GetByPrototype(const MessageLite* prototype, + Arena* arena, + ParseContext* ctx = nullptr) const; + RepeatedPtrFieldBase* MutableByPrototype(const MessageLite* prototype, + Arena* arena, + ParseContext* ctx = nullptr); + + void Clear(); + + // Updates state such that state set in other overwrites this. + // + // Internal Lazy state transitions are updated as such: + // + // src\dest | UNINIT | INIT | DIRTY | CLEAR | ERROR + // :------- | :----: | :---: | :---: | :-----------: | :---: + // UNINIT | DIRTY | DIRTY | DIRTY | UNINIT/DIRTY* | DIRTY + // INIT | DIRTY | DIRTY | DIRTY | UNINIT/DIRTY* | UNDEF + // DIRTY | DIRTY | DIRTY | DIRTY | UNINIT/DIRTY* | UNDEF + // CLEAR | UNINIT | INIT | DIRTY | CLEAR | UNDEF + // ERROR | DIRTY | DIRTY | DIRTY | DIRTY | DIRTY + // * Depends on if clear was initialized before. + // TODO: The state after ERROR should be DIRTY. Also need to make the + // change for LazyField. + void MergeFrom(const MessageLite* prototype, + const LazyRepeatedPtrField& other, Arena* arena, + Arena* other_arena); + + static void Swap(LazyRepeatedPtrField* lhs, Arena* lhs_arena, + LazyRepeatedPtrField* rhs, Arena* rhs_arena); + static void InternalSwap(LazyRepeatedPtrField* lhs, + LazyRepeatedPtrField* rhs); + + const RepeatedPtrFieldBase* TryGetRepeated() const; + + // Returns true when the lazy field has data that have not yet parsed. + // (i.e. parsing has been deferred) Once parsing has been attempted, this + // returns false. Note that the LazyField object may still contain + // the raw unparsed data with parsing errors. + bool HasUnparsed() const { + return GetLogicalState() == LogicalState::kParseRequired; + } + + // Returns true if parsing has been attempted and it failed. + bool HasParsingError() const { + auto raw = raw_.load(std::memory_order_relaxed); + return raw.status() == RawState::kParseError; + } + + // APIs that will be used by table-driven parsing. + // + // `TagType` is passed from table-driven parser. On fast path it's uint8 or + // uint16; on slow path it's uint32. + template + const char* _InternalParse(const MessageLite* prototype, Arena* arena, + const char* ptr, ParseContext* ctx, + TagType expected_tag) { + // If this message is eagerly-verified lazy, kEager mode likely suggests + // that previous verification has failed and we fall back to eager-parsing + // (either to initialize the message to match eager field or to fix false + // errors. + // + // Lazy parsing does not support aliasing and may result in data copying. + // It seems prudent to honor aliasing to avoid any observable gaps between + // lazy and eager parsing. + if (ctx->lazy_parse_mode() == ParseContext::kEager || + ctx->AliasingEnabled()) { + auto* value = MutableByPrototype(prototype, arena, ctx); + ptr = ParseToRepeatedMessage(ptr, ctx, prototype, expected_tag, + value); + return ptr; + } + + switch (GetLogicalState()) { + case LogicalState::kParseRequired: { + return ParseToCord(ptr, ctx, prototype, arena, expected_tag); + } break; + + case LogicalState::kClear: { + // Clear/Fresh have empty unparsed data; so this is the equivalent + // of setting it to the passed in bytes. + return ParseToCord(ptr, ctx, prototype, arena, expected_tag); + } break; + + // Pointers exposed. + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: { + PerformTransition([&](ExclusiveTxn& txn) { + auto* value = txn.mutable_value(); + ptr = ParseToRepeatedMessage(ptr, ctx, prototype, + expected_tag, value); + return RawState::kIsParsed; + }); + return ptr; + } + } + // Required for certain compiler configurations. + internal::Unreachable(); + return nullptr; + } + + template + const char* _InternalParseVerify(const MessageLite* prototype, Arena* arena, + const char* ptr, ParseContext* ctx, + TagType expected_tag, + absl::string_view data) { + ABSL_DCHECK(ptr != nullptr); + if (ctx->lazy_parse_mode() == ParseContext::kLazy || + ctx->lazy_eager_verify_func() == nullptr) { + return ptr; + } + VerifyResult res = WireFormatVerifyView(data, ctx); + if (res.verified) { + if (res.missing_required_fields) { + // Unparsed data may be uninitialized and need to be parsed to be sure. + SetNeedsParseMaybeUninitialized(); + } + return ptr; + } + + // Try eager parsing on potentially malformed wire in case the eager parsing + // fixes the issue. For example, a negative int32 encoded as 5B varint can + // be parsed correctly. + // + // Should preserve the old parsing mode because we don't want to + // unnecessarily eager-parse other parts of message tree. This can be + // especially inefficient if the eager verification results in false + // positive errors. + ParseContext::LazyParseMode old = + ctx->set_lazy_parse_mode(ParseContext::kEager); + (void)GetByPrototype(prototype, arena, ctx); + + // If eager parsing still fails, don't bother restoring the parse mode. + if (HasParsingError()) return nullptr; + + // Unverified lazy fields may miss parsing errors on eager parsing. If it's + // certain, just mark error and return. + if (!ctx->treat_eager_parsing_errors_as_errors()) { + auto raw = raw_.load(std::memory_order_relaxed); + raw.set_status(RawState::kParseError); + raw_.store(raw, std::memory_order_relaxed); + ABSL_DCHECK(HasParsingError()); + return nullptr; + } + + // We need to transition to dirty to prefer eager serialization as the + // unparsed_ has non-canonical wire format. + (void)MutableByPrototype(prototype, arena); + + (void)ctx->set_lazy_parse_mode(old); + return ptr; + } + + template + static const char* ParseToRepeatedMessage(const char* ptr, ParseContext* ctx, + const MessageLite* prototype, + TagType expected_tag, + RepeatedPtrFieldBase* value) { + const char* ptr2 = ptr; + TagType next_tag; + do { + MessageLite* submsg = value->AddMessage(prototype); + // ptr2 points to the start of the element's encoded length. + ptr = ctx->ParseMessage(submsg, ptr2); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + if (ctx->Done(&ptr)) { + break; + } + } + ptr2 = ReadTagInternal(ptr, &next_tag); + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) return nullptr; + } while (next_tag == expected_tag); + return ptr; + } + + template + const char* ParseToCord(const char* ptr, ParseContext* ctx, + const MessageLite* prototype, Arena* arena, + TagType expected_tag) { + // ptr2 points to the start of the encoded length. + const char* ptr2 = ptr; + TagType next_tag; + // Move ptr back to the start of the tag. + size_t tag_size = TagSizeInternal(expected_tag); + ptr -= tag_size; + if (ctx->parent_missing_required_fields()) { + SetNeedsParseMaybeUninitialized(); + } else { + SetNeedsParse(); + } + do { + std::string tmp; + // Append the tag. + tmp.append(absl::string_view(ptr, ptr2 - ptr)); + size_t taglen_size; + ptr = ctx->ParseLengthDelimitedInlined( + ptr2, [&tmp, &taglen_size, ctx, ptr2](const char* p) { + // At this moment length is read and p points to the start of + // the payload. + ABSL_DCHECK(p - ptr2 > 0 && p - ptr2 <= 5) << p - ptr2; + // Append the length. + tmp.append(absl::string_view(ptr2, p - ptr2)); + taglen_size = tmp.size(); + return ctx->AppendString(p, &tmp); + }); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + const auto tmp_size = tmp.size(); + ABSL_DCHECK_GE(tmp_size, taglen_size); + if (unparsed_.IsCord()) { + unparsed_.AsCord().Append(tmp); + } else if (arena != nullptr && + unparsed_.Size() + tmp_size <= kMaxArraySize) { + if (unparsed_.IsEmpty()) { + unparsed_.InitAsArray(arena, 0); + } + unparsed_.AppendToArray(tmp); + } else { + unparsed_.UpgradeToCord(arena).Append(tmp); + } + if (tmp_size > taglen_size) { + ptr = _InternalParseVerify( + prototype, arena, ptr, ctx, expected_tag, + absl::string_view(tmp.data() + taglen_size, + tmp_size - taglen_size)); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + } + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + // `Done` advances the stream to the next buffer chunk. + if (ctx->Done(&ptr)) { + break; + } + } + // ptr points to the start of the next tag. + ptr2 = ReadTagInternal(ptr, &next_tag); + // ptr2 points to the start of the next element's encoded length. + + // TODO: Try to remove the following condition for 8 and 16 bits + // TagType. + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) return nullptr; + } while (next_tag == expected_tag); + if (unparsed_.IsArray()) { + unparsed_.ZeroOutTailingBytes(); + } + return ptr; + } + + uint8_t* InternalWrite(const MessageLite* prototype, int32_t number, + uint8_t* target, + io::EpsCopyOutputStream* stream) const; + + // ByteSize of the repeated ptr field (including the varints of tags and + // lengths). + size_t ByteSizeLong(size_t tag_size) const; + size_t SpaceUsedExcludingSelfLong() const; + + // LogicalState combines the `raw_` and `unparsed_` fields to produce the + // current state. + // + // This separation allows more easily adding fine-grained states w/o touching + // std::atomics; most state transitions are in a write context and do not + // require subtle atomicity. + // TODO: Deduplicate with LazyField. + enum class LogicalState { + // The serialized data is available and unparsed. + // (kParseRequired, !unparsed.empty(), message = undefined). + kParseRequired, + // The message has been parsed from the serialized data. + // (kIsParsed, !unparsed.empty(), message != nullptr). + kNoParseRequired, + // The field is clear (freshly constructed or cleared): + // - (kCleared, unparsed.empty(), message = nullptr) + kClear, + // The field is clear but previously exposed a pointer. + // - (kCleared, unparsed.empty(), message = !nullptr) + kClearExposed, + // A write operation was done after a parse. + // (kIsParsed, unparsed.empty(), message != nullptr) + kDirty, + }; + LogicalState GetLogicalState() const { + auto raw = raw_.load(std::memory_order_acquire); + switch (raw.status()) { + case RawState::kParseError: + ABSL_DCHECK_NE(raw.value(), nullptr); + return LogicalState::kDirty; + case RawState::kCleared: + ABSL_DCHECK(unparsed_.IsEmpty()); + ABSL_DCHECK(raw.value() == nullptr || raw.value()->empty()) + << (raw.value() == nullptr + ? "nullptr" + : absl::StrCat("non-empty:", raw.value()->size())); + return raw.value() == nullptr ? LogicalState::kClear + : LogicalState::kClearExposed; + case RawState::kNeedsParse: + case RawState::kNeedsParseMaybeUninitialized: + // There is no SetEncoded, so unparsed_ is always from _InternalParse, + // which can't be empty. + ABSL_DCHECK(!unparsed_.IsEmpty()); + ABSL_DCHECK(raw.value() == nullptr || raw.value()->empty()); + return LogicalState::kParseRequired; + default: + ABSL_DCHECK(raw.status() == RawState::kIsParsed || + raw.status() == RawState::kIsParsedMaybeUninitialized); + ABSL_DCHECK(raw.value() != nullptr); + // Only other Initialized state was kParseError which is handled above. + if (unparsed_.IsEmpty()) { + return LogicalState::kDirty; + } + // Non-null message, unparsed exists. + return LogicalState::kNoParseRequired; + } + } + + private: + // Values that can be kept in `MessageState`'s status bits. + // TODO: Deduplicate with LazyField. + enum class RawState { + // `unparsed_` is empty. + // `message_` is either nullptr or an empty container. + kCleared, + + // `unparsed_` contains the canonical field data. + // `message_` points to the result of parsing that data. + // + // NOTE: serializing `message_` may produce different bytes than + // `unparsed_`, so care must be taken around issues of canonical or + // deterministic serialization. Generally, `unparsed_` should be preferred + // if it is not empty, as that is lower overhead. + kIsParsed, + + // IsParsed and may be uninitialized. See + // kNeedsParseMaybeUninitialized for details. + kIsParsedMaybeUninitialized, + + // TODO: add kIsParsedIgnoreUnparsed and + // kIsParsedIgnoreUnparsedMaybeUninitialized. + + // `message_` points to the result of parsing that data, but there was an + // error when parsing. Partially parsed `message_` is considered canonical + // to match eager fields. + kParseError, + + // `unparsed_` contains the field data. + // `message_` is either nullptr or an empty container. + kNeedsParse, + + // kNeedsParse and may be uninitialized. + // + // MaybeUninitialized is flagged in the verification and recorded to trigger + // eager parsing on IsInitialized() to be certain. + // + // Note that unverified data is assumed to be initialized (to support legacy + // cases) and treated as if it's verified to be initialized. Therefore, we + // need "MaybeUninitialized" rather than "Initialized". + kNeedsParseMaybeUninitialized, + + kMaxState = kNeedsParseMaybeUninitialized + }; + + class MessageState { + public: + constexpr explicit MessageState(RawState state) : raw_(ToUint32(state)) {} + MessageState(const RepeatedPtrFieldBase* message, RawState state) + : raw_(reinterpret_cast(message) | ToUint32(state)) { + ABSL_DCHECK_EQ(reinterpret_cast(message) & ToUint32(state), + 0u); + } + + const RepeatedPtrFieldBase* value() const { + return reinterpret_cast(raw_ & ~0b111); + } + + RepeatedPtrFieldBase* mutable_value() const { + return reinterpret_cast(raw_ & ~0b111); + } + + RawState status() const { return ToRawState(raw_ & 0b111); } + + void set_status(RawState status) { + raw_ &= ~0b111; + raw_ |= ToUint32(status); + } + + void set_value(const RepeatedPtrFieldBase* message) { + raw_ &= 0b111; + raw_ |= reinterpret_cast(message); + } + + static inline constexpr uint32_t ToUint32(RawState status) { + return static_cast(status); + } + static inline RawState ToRawState(uint32_t status) { + ABSL_DCHECK_LE(status, ToUint32(RawState::kMaxState)); + return static_cast(status); + } + + bool NeedsParse() const { + // kNeedsParse and kNeedsParseMaybeUninitialized must be 0 and 1 to make + // NeedsParse() check cheap. + static_assert( + RawState::kNeedsParseMaybeUninitialized == RawState::kMaxState, ""); + static_assert(ToUint32(RawState::kNeedsParseMaybeUninitialized) == + ToUint32(RawState::kNeedsParse) + 1, + ""); + return status() >= RawState::kNeedsParse; + } + + private: + uintptr_t raw_; + }; + + // TODO: Deduplicate. + template + class ByTemplate { + public: + // Only `Get()` needs access to the default element, but we don't want to + // force instantiation of `MessageType::default_instance()` because it + // doesn't exist in all configurations. + explicit ByTemplate() : ByTemplate(nullptr) {} + explicit ByTemplate(const MessageType* default_instance) + : default_instance_(default_instance) {} + + MessageLite* New(Arena* arena) const { + return reinterpret_cast( + Arena::DefaultConstruct(arena)); + } + + const MessageLite& Default() const { + ABSL_DCHECK(default_instance_ != nullptr); + return *reinterpret_cast(default_instance_); + } + + private: + const MessageType* default_instance_; + }; + + // Copy constructor on arena. + LazyRepeatedPtrField(Arena* arena, const LazyRepeatedPtrField& rhs, + Arena* rhs_arena); + + // Serialization methods. Note that WriteToCord may override/clear the + // given cord. + template + bool MergeFrom(const MessageLite* prototype, const Input& data, Arena* arena); + + private: + template + MessageState SharedInit(Strategy strategy, Arena* arena, + ParseContext* ctx) const { + auto old_raw = raw_.load(std::memory_order_acquire); + if (!old_raw.NeedsParse()) return old_raw; + MessageState new_raw = + // Transfer MaybeUninitialized state after a state transition. + DoParse(nullptr, strategy.Default(), arena, ctx, + old_raw.status() == RawState::kNeedsParseMaybeUninitialized); + if (raw_.compare_exchange_strong(old_raw, new_raw, + std::memory_order_release, + std::memory_order_acquire)) { + // We won the race. Dispose of the old message (if there was one). + if (arena == nullptr) { + delete reinterpret_cast*>( + old_raw.value()); + } + return new_raw; + } else { + // We lost the race, but someone else will have installed the new + // value. Dispose of the our attempt at installing. + if (arena == nullptr) { + delete reinterpret_cast*>( + new_raw.value()); + } + ABSL_DCHECK(!old_raw.NeedsParse()); + return old_raw; + } + } + + template + MessageState ExclusiveInitWithoutStore(Strategy strategy, Arena* arena, + ParseContext* ctx) { + auto old_raw = raw_.load(std::memory_order_relaxed); + if (!old_raw.NeedsParse() && old_raw.value() != nullptr) return old_raw; + if (old_raw.NeedsParse()) { + // Mutable messages need not transfer MaybeUninitialized. + return DoParse(old_raw.mutable_value(), strategy.Default(), arena, ctx, + false); + } + ABSL_DCHECK(old_raw.value() == nullptr); + return MessageState(Arena::Create(arena), + RawState::kIsParsed); + } + + template + const RepeatedPtrFieldBase* GetGeneric(Strategy strategy, Arena* arena, + ParseContext* ctx) const { + const auto* value = SharedInit(strategy, arena, ctx).value(); + if (value == nullptr) { + return reinterpret_cast(DefaultRawPtr()); + } + return value; + } + + template + RepeatedPtrFieldBase* MutableGeneric(Strategy strategy, Arena* arena, + ParseContext* ctx) { + auto raw = ExclusiveInitWithoutStore(strategy, arena, ctx); + unparsed_.Clear(); + ABSL_DCHECK(raw.value() != nullptr); + raw.set_status(RawState::kIsParsed); + raw_.store(raw, std::memory_order_relaxed); + return raw.mutable_value(); + } + + void SetNeedsParse() { + auto raw = raw_.load(std::memory_order_relaxed); + raw.set_status(RawState::kNeedsParse); + raw_.store(raw, std::memory_order_relaxed); + } + + void SetNeedsParseMaybeUninitialized() { + auto raw = raw_.load(std::memory_order_relaxed); + ABSL_DCHECK(raw.status() == RawState::kNeedsParse || + raw.status() == RawState::kNeedsParseMaybeUninitialized); + raw.set_status(RawState::kNeedsParseMaybeUninitialized); + raw_.store(raw, std::memory_order_relaxed); + } + + void SetParseNotRequiredMaybeUninitialized() { + auto raw = raw_.load(std::memory_order_relaxed); + ABSL_DCHECK(raw.status() == RawState::kIsParsed || + raw.status() == RawState::kIsParsedMaybeUninitialized); + raw.set_status(RawState::kIsParsedMaybeUninitialized); + raw_.store(raw, std::memory_order_relaxed); + } + + bool MaybeUninitialized() const { + auto raw = raw_.load(std::memory_order_relaxed); + if (raw.status() == RawState::kNeedsParseMaybeUninitialized) return true; + + // Make sure the logical state matches as well. + return raw.status() == RawState::kIsParsedMaybeUninitialized && + GetLogicalState() == LogicalState::kNoParseRequired; + } + + // Adds MaybeUninitialized state if "other" may be uninitialized. + void MergeMaybeUninitializedState(const LazyRepeatedPtrField& other); + + bool IsEagerSerializeSafe(const MessageLite* prototype, int32_t number, + Arena* arena) const; + + static void swap_atomics(std::atomic& lhs, + std::atomic& rhs); + + // Helper to enforce invariants when exclusive R/M/W access is required. + class ExclusiveTxn { + public: + explicit ExclusiveTxn(LazyRepeatedPtrField& lazy) + : lazy_(lazy), state_(lazy_.raw_.load(std::memory_order_relaxed)) {} + + RepeatedPtrFieldBase* mutable_value() { + // Any write to the message at this point should nuke unparsed_. + lazy_.unparsed_.Clear(); + return state_.mutable_value(); + } + + void Commit(RawState new_status) { + if (state_.status() != new_status) { + state_.set_status(new_status); + lazy_.raw_.store(state_, std::memory_order_relaxed); + } + } + + private: + LazyRepeatedPtrField& lazy_; + MessageState state_; + }; + + template + RawState PerformTransition(Transition fn) { + ExclusiveTxn txn(*this); + RawState new_state = fn(txn); + txn.Commit(new_state); + return new_state; + } + + public: + // Payload abstraction that can hold a raw char array or a Cord depending on + // how much data it needs to hold. + // The caller is responsible for managing the lifetime of the payload. + // TODO: Deduplicate with the LazyField::UnparsedPayload. + class UnparsedPayload { + enum Tag : uintptr_t { + kTagEmpty = 0, + kTagArray = 1, + kTagCord = 2, + + kTagBits = 3, + kRemoveMask = ~kTagBits, + }; + + public: + using ArraySizeType = uint16_t; + + // Visit the payload and calls the respective callback. The signatures are: + // - () for kUnset + // - (Cord&) for kCord + // - (absl::string_view) for kArray + // Returns the value returned by the callback. + template + auto Visit(UnsetF unset_f, CordF cord_f, ViewF view_f) const { + Tag t = tag(); + // Using ternary to allow for common-type implicit conversions. + return t == kTagEmpty ? unset_f() + : t == kTagArray ? view_f(AsStringView()) + : cord_f(AsCord()); + } + + Tag tag() const { return static_cast(value_ & kTagBits); } + + bool IsCord() const { + ABSL_DCHECK_EQ(static_cast(value_ & kTagCord), + static_cast(tag() == kTagCord)); + return (value_ & kTagCord) != 0u; + } + + bool IsArray() const { + ABSL_DCHECK_EQ(static_cast(value_ & kTagArray), + static_cast(tag() == kTagArray)); + return (value_ & kTagArray) != 0u; + } + + // Requires: IsCord() + absl::Cord& AsCord() const { + ABSL_DCHECK(IsCord()); + return *reinterpret_cast(value_ & kRemoveMask); + } + + // Return the payload as Cord regardless of the existing storage. + absl::Cord ForceAsCord() const { + return Visit([] { return absl::Cord(); }, // + [](const auto& c) { return c; }, + [](auto view) { return absl::Cord(view); }); + } + + // Similar to AsCord(), but if the payload is not already a Cord it will + // convert it first, maintaining existing bytes. + absl::Cord& UpgradeToCord(Arena* arena) { + if (IsCord()) return AsCord(); + absl::Cord new_cord(AsStringView()); + return InitAsCord(arena, std::move(new_cord)); + } + + // Requires: input array is the untagged value. + ArraySizeType GetArraySize(const char* array) const { + ABSL_DCHECK_EQ(array, reinterpret_cast(value_ - kTagArray)); + ArraySizeType size; + memcpy(&size, array, sizeof(size)); + return size; + } + + void SetArraySize(void* array, ArraySizeType size) const { + ABSL_DCHECK_EQ(array, reinterpret_cast(value_ - kTagArray)); + memcpy(array, &size, sizeof(ArraySizeType)); + } + + void SetArraySize(ArraySizeType size) const { + void* array = reinterpret_cast(value_ - kTagArray); + memcpy(array, &size, sizeof(ArraySizeType)); + } + + // Requires: !IsCord() + absl::string_view AsStringView() const { + switch (tag()) { + case kTagEmpty: + return {}; + + case kTagArray: { + const char* array = reinterpret_cast(value_ - kTagArray); + auto size = GetArraySize(array); + return absl::string_view(array + sizeof(size), size); + } + + default: + Unreachable(); + } + } + + // Clear the payload. After this call `Size()==0` and `IsEmpty()==true`, but + // it is not necessarily true that `tag()==kTagEmpty`. + // In particular, it keeps the Cord around in case it needs to be reused. + void Clear() { + switch (tag()) { + case kTagEmpty: + case kTagArray: + value_ = 0; + break; + default: + AsCord().Clear(); + break; + } + } + + // Destroys allocated memory if necessary. Does not reset the object. + void Destroy() { + if (IsCord()) delete &AsCord(); + } + + bool IsEmpty() const { + return Visit([] { return true; }, + [](const auto& cord) { return cord.empty(); }, + [](auto view) { + ABSL_DCHECK(!view.empty()); + return false; + }); + } + + size_t Size() const { + return Visit([] { return 0; }, + [](const auto& cord) { return cord.size(); }, + [](auto view) { return view.size(); }); + } + + // Sets the currently value as a Cord constructed from `args...`. + // It will clean up the existing value if necessary. + template + void SetCord(Arena* arena, Arg&& arg) { + if (IsCord()) { + // Reuse the existing cord. + AsCord() = std::forward(arg); + } else { + absl::Cord* cord = + Arena::Create(arena, std::forward(arg)); + value_ = reinterpret_cast(cord) | kTagCord; + } + } + + // Initialize the value as a Cord constructed from `args...` + // Ignores existing value. + template + absl::Cord& InitAsCord(Arena* arena, Args&&... args) { + auto* cord = + Arena::Create(arena, std::forward(args)...); + value_ = reinterpret_cast(cord) | kTagCord; + return *cord; + } + + // Initialize the value as an array copied from `view`. The tailing bytes + // are set to 0 to avoid UB. + // Ignores existing value. + void InitAndSetArray(Arena* arena, absl::string_view view) { + char* array = InitAsArray(arena, view.size()); + memcpy(array, view.data(), view.size()); + if (view.size() < kMaxArraySize) { + // Memset uninit data to avoid UB later. + memset(array + view.size(), '\0', kMaxArraySize - view.size()); + } + ABSL_DCHECK_EQ(view, AsStringView()); + } + + // Initialize the value as an array copied from `cord`. The tailing bytes + // are set to 0 to avoid UB. + // Ignores existing value. + void InitAndSetArray(Arena* arena, const absl::Cord& cord) { + auto size = cord.size(); + char* array = InitAsArray(arena, size); + cord.CopyToArray(array); + if (size < kMaxArraySize) { + // Memset uninit data to avoid UB later. + memset(array + size, '\0', kMaxArraySize - size); + } + } + + // Initialize the value as an array of size `size`. The payload bytes are + // uninitialized. + // Ignores existing value. + char* InitAsArray(Arena* arena, ArraySizeType size) { + ABSL_DCHECK(arena != nullptr); + // Allocate max allowed capacity. + // TODO: improve this to reduce waste when the size is small. + void* c = arena->AllocateAligned(kMaxArraySize + sizeof(ArraySizeType)); + ABSL_DCHECK_EQ(reinterpret_cast(c) & kTagBits, uintptr_t{0}); + value_ = reinterpret_cast(c) | kTagArray; + SetArraySize(c, size); + return static_cast(c) + sizeof(ArraySizeType); + } + + void AppendToArray(absl::string_view view) { + char* array = reinterpret_cast(value_ - kTagArray); + ArraySizeType size = GetArraySize(array); + char* c = array + sizeof(size) + size; + size += view.size(); + SetArraySize(array, size); + memcpy(c, view.data(), view.size()); + } + + void ZeroOutTailingBytes() { + char* array = reinterpret_cast(value_ - kTagArray); + auto size = GetArraySize(array); + if (size < kMaxArraySize) { + memset(array + sizeof(ArraySizeType) + size, '\0', + kMaxArraySize - size); + } + } + + size_t SpaceUsedExcludingSelf() const { + return Visit( + [] { return 0; }, + [](const auto& cord) { return cord.EstimatedMemoryUsage(); }, + [](auto view) { return kMaxArraySize + sizeof(ArraySizeType); }); + } + + void TransferHeapOwnershipToArena(Arena* arena) { + ABSL_DCHECK(tag() == kTagCord || tag() == kTagEmpty); + if (IsCord()) arena->Own(&AsCord()); + } + + private: + uintptr_t value_ = 0; + }; + + public: + static bool ParseWithOuterContext(RepeatedPtrFieldBase* value, + const absl::Cord& input, ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required); + static bool ParseWithOuterContext(RepeatedPtrFieldBase* value, + absl::string_view input, ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required); + + private: + // This method has to be below the definition of class UnparsedPayload due to + // the call to `unparsed_.Visit`. + // TODO: Deduplicate with LazyField. + MessageState DoParse(RepeatedPtrFieldBase* old, const MessageLite& prototype, + Arena* arena, ParseContext* ctx, + bool maybe_uninitialized) const { + auto* value = + (old == nullptr) ? Arena::Create(arena) : old; + if (!unparsed_.Visit( + [] { return true; }, + [&](const auto& cord) { + return ParseWithOuterContext(value, cord, ctx, &prototype, + maybe_uninitialized); + }, + [&](auto view) { + return ParseWithOuterContext(value, view, ctx, &prototype, + maybe_uninitialized); + })) { + // If this is called by eager verficiation, ctx != nullptr and logging + // parsing error in that case is likely redundant because the parsing will + // fail anyway. Users who care about parsing errors would have already + // checked the return value and others may find the error log unexpected. + // + // `ctx == nullptr` means it's not eagerly verified (e.g. unverified lazy) + // and logging in that case makes sense. + if (ctx == nullptr) { + LogParseError(value); + } + return MessageState(value, RawState::kParseError); + } + return MessageState(value, maybe_uninitialized + ? RawState::kIsParsedMaybeUninitialized + : RawState::kIsParsed); + } + + // Mutable because it is initialized lazily. + // A MessageState is a tagged RepeatedPtrFieldBase* + mutable std::atomic raw_; + + // NOT mutable because we keep the payload around until the message changes in + // some way. + UnparsedPayload unparsed_; + // absl::Cord will make copies on anything under this limit, so we might as + // well do the copies into our own buffer instead. + static constexpr size_t kMaxArraySize = 512; + static_assert(kMaxArraySize <= + std::numeric_limits::max()); + friend class ::google::protobuf::Arena; + friend class ::google::protobuf::Reflection; + friend class ExtensionSet; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + + // Logs a parsing error. + static void LogParseError(const RepeatedPtrFieldBase* value); + + bool IsAllocated() const { + return raw_.load(std::memory_order_acquire).value() != nullptr; + } + + // For testing purposes. + friend class LazyRepeatedPtrFieldTest; + friend class LazyRepeatedInMessageTest; + template + void OverwriteForTest(RawState status, const absl::Cord& unparsed, + RepeatedPtrField* value, Arena* arena); +}; + +inline LazyRepeatedPtrField::~LazyRepeatedPtrField() { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + delete reinterpret_cast*>(value); + unparsed_.Destroy(); +} + +// TODO: Deduplicate with LazyField. +inline const RepeatedPtrFieldBase* LazyRepeatedPtrField::TryGetRepeated() + const { + switch (GetLogicalState()) { + case LogicalState::kDirty: + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: + return raw_.load(std::memory_order_relaxed).value(); + case LogicalState::kClear: + case LogicalState::kClearExposed: + return nullptr; + } + internal::Unreachable(); + return nullptr; +} + +// ------------------------------------------------------------------- +// Testing stuff. + +// It's in the header due to the template. +// TODO: Deduplicate with LazyField. +template +void LazyRepeatedPtrField::OverwriteForTest(RawState status, + const absl::Cord& unparsed, + RepeatedPtrField* value, + Arena* arena) { + auto raw = raw_.load(std::memory_order_relaxed); + if (arena == nullptr) { + delete reinterpret_cast*>(raw.value()); + } + raw.set_value(reinterpret_cast(value)); + raw.set_status(status); + if (!unparsed.empty()) { + if (arena != nullptr && unparsed.size() <= kMaxArraySize) { + unparsed_.InitAndSetArray(arena, unparsed); + } else { + unparsed_.SetCord(arena, unparsed); + } + } + raw_.store(raw, std::memory_order_relaxed); +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" + +#endif // GOOGLE_PROTOBUF_LAZY_REPEATED_FIELD_H__ diff --git a/src/google/protobuf/lazy_repeated_field_heavy.cc b/src/google/protobuf/lazy_repeated_field_heavy.cc new file mode 100644 index 0000000000..1c4e1bf686 --- /dev/null +++ b/src/google/protobuf/lazy_repeated_field_heavy.cc @@ -0,0 +1,401 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/strings/cord.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/lazy_repeated_field.h" +#include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" +#include "google/protobuf/repeated_ptr_field.h" +#include "google/protobuf/wire_format_lite.h" + +namespace google { +namespace protobuf { +namespace internal { +namespace { + +class ByFactory { + public: + explicit ByFactory(const Descriptor* type, MessageFactory* factory) + : type_(type), factory_(factory) {} + + Message* New(Arena* arena) const { + return factory_->GetPrototype(type_)->New(arena); + } + + const Message& Default() const { return *factory_->GetPrototype(type_); } + + private: + const Descriptor* type_; + MessageFactory* factory_; +}; + +// Escape C++ trigraphs by escaping question marks to \? +std::string EscapeTrigraphs(absl::string_view to_escape) { + return absl::StrReplaceAll(to_escape, {{"?", "\\?"}}); +} + +std::string EscapeEncoded(absl::string_view encoded) { + std::string out; + out.reserve(encoded.size() * 2); + constexpr size_t kBytesPerLine = 25; + for (size_t i = 0; i < encoded.size(); i += kBytesPerLine) { + absl::StrAppend( + &out, "\"", + EscapeTrigraphs(absl::CEscape(encoded.substr(i, kBytesPerLine))), + "\"\n"); + } + return out; +} + +// Deterministic serialization is required to minimize false positives: e.g. +// ordering, redundant wire format data, etc. Such discrepancies are +// expected and tolerated. To prevent this serialization starts yet another +// consistency check, we should skip consistency-check. +std::string DeterministicSerialization(const google::protobuf::MessageLite& m) { + std::string result; + { + google::protobuf::io::StringOutputStream sink(&result); + google::protobuf::io::CodedOutputStream out(&sink); + out.SetSerializationDeterministic(true); + out.SkipCheckConsistency(); + m.SerializePartialToCodedStream(&out); + } + return result; +} + +// If LazyField is initialized, unparsed and message should be consistent. If +// a LazyField is mutated via const_cast, that may break. We should rather fail +// than silently propagate such discrepancy. Note that this aims to detect +// missing/added data. +void VerifyConsistency(LazyRepeatedPtrField::LogicalState state, + const RepeatedPtrFieldBase* value, + const MessageLite* prototype, const absl::Cord& unparsed, + io::EpsCopyOutputStream* stream) { +#ifndef NDEBUG + if (stream != nullptr && !stream->ShouldCheckConsistency()) return; + if (state != LazyRepeatedPtrField::LogicalState::kNoParseRequired) return; + + RepeatedPtrField unparsed_msg; + if (!LazyRepeatedPtrField::ParseWithOuterContext( + reinterpret_cast(&unparsed_msg), unparsed, + nullptr, prototype, /*set_missing_required=*/false)) { + // Bail out on parse failure as it can result in false positive + // inconsistency and ABSL_CHECK failure. Warn instead. + ABSL_LOG(WARNING) + << "Verify skipped due to parse falure: RepeatedPtrField of " + << prototype->GetTypeName(); + return; + } + + const auto* msgs = reinterpret_cast*>(value); + // Eagerly parse all lazy fields to eliminate non-canonical wireformat data. + for (int i = 0; i < msgs->size(); i++) { + // Clone a new one from the original to eagerly parse all lazy + // fields. + const auto& msg = msgs->Get(i); + std::unique_ptr clone(msg.New()); + clone->CopyFrom(msg); + EagerParseLazyFieldIgnoreUnparsed(*clone); + EagerParseLazyFieldIgnoreUnparsed(*unparsed_msg.Mutable(i)); + ABSL_DCHECK_EQ(DeterministicSerialization(*clone), + DeterministicSerialization(unparsed_msg.Get(i))) + << "RepeatedPtrField<" << msg.GetTypeName() << ">(" << i << ")" + << ": likely mutated via getters + const_cast\n" + << "unparsed:\n" + << EscapeEncoded(DeterministicSerialization(unparsed_msg.Get(i))) + << "\nmessage:\n" + << EscapeEncoded(DeterministicSerialization(*clone)); + } +#endif // !NDEBUG +} + +} // namespace + +LazyRepeatedPtrField::LazyRepeatedPtrField(Arena* arena, + const LazyRepeatedPtrField& rhs, + Arena* rhs_arena) + : raw_(MessageState(RawState::kCleared)) { + switch (rhs.GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + return; // Leave uninitialized / empty + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: { + rhs.unparsed_.Visit( + [] {}, // + [&](const auto& cord) { unparsed_.InitAsCord(arena, cord); }, + [&](auto view) { + if (arena == nullptr) { + unparsed_.InitAsCord(nullptr, view); + } else { + unparsed_.InitAndSetArray(arena, view); + } + }); + raw_.store( + MessageState(nullptr, rhs.MaybeUninitialized() + ? RawState::kNeedsParseMaybeUninitialized + : RawState::kNeedsParse), + std::memory_order_relaxed); + return; + } + case LogicalState::kDirty: { + MessageState state = rhs.raw_.load(std::memory_order_relaxed); + const auto* src = state.value(); + if (src->empty()) { + return; // Leave uninitialized / empty + } + // Retain the existing IsParsed or IsParsedMaybeUninitialized status. + // TODO: use copy construction. + auto new_state = state.status(); + auto* value = Arena::Create(arena); + // MergeFrom calls reserve. + value->MergeFrom(*src); + raw_.store(MessageState(value, new_state), std::memory_order_relaxed); + return; + } + } +} + +const RepeatedPtrFieldBase* LazyRepeatedPtrField::GetDynamic( + const Descriptor* type, MessageFactory* factory, Arena* arena) const { + return GetGeneric(ByFactory(type, factory), arena, nullptr); +} + +RepeatedPtrFieldBase* LazyRepeatedPtrField::MutableDynamic( + const Descriptor* type, MessageFactory* factory, Arena* arena) { + return MutableGeneric(ByFactory(type, factory), arena, nullptr); +} + +size_t LazyRepeatedPtrField::SpaceUsedExcludingSelfLong() const { + // absl::Cord::EstimatedMemoryUsage counts itself that should be excluded + // because sizeof(Cord) is already counted in self. + size_t total_size = unparsed_.SpaceUsedExcludingSelf(); + switch (GetLogicalState()) { + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + total_size += + value->SpaceUsedExcludingSelfLong>(); + } break; + case LogicalState::kClear: + case LogicalState::kParseRequired: + // We may have a `Message*` here, but we cannot safely access it + // because, a racing SharedInit could delete it out from under us. + // Other states in this structure are already passed kSharedInit and are + // thus safe. + break; // Nothing to add. + } + return total_size; +} + +template +bool LazyRepeatedPtrField::MergeFrom(const MessageLite* prototype, + const Input& data, Arena* arena) { + switch (GetLogicalState()) { + case LogicalState::kParseRequired: { + unparsed_.UpgradeToCord(arena).Append(data); + break; + } + case LogicalState::kClear: { + size_t num_bytes = data.size(); + ABSL_DCHECK(num_bytes > 0); + if (arena == nullptr || num_bytes > kMaxArraySize || unparsed_.IsCord()) { + unparsed_.SetCord(arena, data); + } else { + unparsed_.InitAndSetArray(arena, data); + } + SetNeedsParse(); + break; + } + + // Pointer was previously exposed merge into that object. + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: { + auto new_state = PerformTransition([&](ExclusiveTxn& txn) { + auto* value = txn.mutable_value(); + bool res = + ParseWithOuterContext(value, data, /*ctx=*/nullptr, prototype, + /*set_missing_required=*/false); + if (!res) { + LogParseError(value); + return RawState::kParseError; + } else { + return RawState::kIsParsed; + } + }); + return new_state == RawState::kIsParsed; + } + } + return true; +} + +void LazyRepeatedPtrField::MergeMaybeUninitializedState( + const LazyRepeatedPtrField& other) { + if (MaybeUninitialized() || !other.MaybeUninitialized()) return; + + switch (GetLogicalState()) { + case LogicalState::kParseRequired: + SetNeedsParseMaybeUninitialized(); + break; + case LogicalState::kNoParseRequired: + SetParseNotRequiredMaybeUninitialized(); + break; + default: + break; + } +} + +void LazyRepeatedPtrField::MergeFrom(const MessageLite* prototype, + const LazyRepeatedPtrField& other, + Arena* arena, Arena* other_arena) { +#ifndef NDEBUG + VerifyConsistency(other.GetLogicalState(), + other.raw_.load(std::memory_order_relaxed).value(), + prototype, other.unparsed_.ForceAsCord(), nullptr); +#endif // !NDEBUG + switch (other.GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + return; // Nothing to do. + + case LogicalState::kParseRequired: + case LogicalState::kNoParseRequired: + if (other.unparsed_.IsCord()) { + MergeFrom(prototype, other.unparsed_.AsCord(), arena); + } else { + MergeFrom(prototype, other.unparsed_.AsStringView(), arena); + } + MergeMaybeUninitializedState(other); + return; + + case LogicalState::kDirty: { + const auto* other_value = + other.raw_.load(std::memory_order_relaxed).value(); + if (other_value->empty()) { + return; + } + auto* value = MutableByPrototype(prototype, arena); + value->MergeFrom(*other_value); + // No need to merge uninitialized state. + ABSL_DCHECK(GetLogicalState() == LogicalState::kDirty); + return; + } + } +} + +uint8_t* LazyRepeatedPtrField::InternalWrite( + const MessageLite* prototype, int32_t number, uint8_t* target, + io::EpsCopyOutputStream* stream) const { +#ifndef NDEBUG + VerifyConsistency(GetLogicalState(), + raw_.load(std::memory_order_relaxed).value(), prototype, + unparsed_.ForceAsCord(), stream); +#endif // !NDEBUG + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: + // If deterministic is enabled then attempt to parse to a message which + // can then be serialized deterministically. (The serialized bytes may + // have been created undeterministically). + if (stream->IsSerializationDeterministic() && prototype != nullptr) { + RepeatedPtrField value; + // TODO: Test this path. + bool success = unparsed_.Visit( + [] { return true; }, + [&](const auto& cord) { + // `set_missing_required = false` to avoid checking require fields + // (simialr to Message::ParsePartial*). + return ParseWithOuterContext( + reinterpret_cast(&value), cord, + /*ctx=*/nullptr, prototype, /*set_missing_required=*/false); + }, + [&](auto view) { + return ParseWithOuterContext( + reinterpret_cast(&value), view, + /*ctx=*/nullptr, prototype, false); + }); + if (success) { + size_t tag_size = WireFormatLite::TagSize( + number, WireFormatLite::FieldType::TYPE_MESSAGE); + auto count = tag_size * value.size(); + for (int i = 0; i < value.size(); i++) { + count += WireFormatLite::LengthDelimitedSize( + value.Get(i).ByteSizeLong()); + } + + // Serialization takes place in two phases: + // 1) Figure out the expected number of bytes (e.g. ByteSizeLong() on + // the container message) 2) InternalWrite the bytes. + // + // There is a golden contract that the # of bytes written matches + // the returned value from the first step. + // + // In this case unparsed_ was used as the source of truth for the + // number of bytes. There are some known cases where the number of + // serialized bytes is different than the number of bytes written + // by a message parsed from the serialized bytes. For example if the + // serialized representation contained multiple entries for the same + // non-repeated field the duplicates are removed upon parsing. + // + // If this (relatively rare) case is hit then there is no choice + // but to serialize the original unparsed bytes; otherwise the + // golden contract is broken. + // It's possible for the size to change if the unparsed_ was not + // canonical, for example it can have repeated entries for the same + // tag (this is more common then you would think). + if (count == unparsed_.Size()) { + for (int i = 0, n = value.size(); i < n; i++) { + const auto& repfield = value.Get(i); + target = WireFormatLite::InternalWriteMessage( + number, repfield, repfield.GetCachedSize(), target, stream); + } + return target; + } + } + } + return unparsed_.Visit( + [&] { return target; }, + [&](const auto& cord) { return stream->WriteCord(cord, target); }, + [&](auto view) { + return stream->WriteRaw(view.data(), view.size(), target); + }); + case LogicalState::kDirty: { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + for (int i = 0, n = value->size(); i < n; i++) { + const auto& repfield = value->Get>(i); + target = WireFormatLite::InternalWriteMessage( + number, repfield, repfield.GetCachedSize(), target, stream); + } + return target; + } + } + // Required for certain compiler configurations. + ABSL_LOG(FATAL) << "Not reachable"; + return nullptr; +} + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index 9dfa8e8801..fcd99edc1b 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -576,6 +576,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { // subclass. friend class google::protobuf::Reflection; friend class internal::SwapFieldHelper; + friend class LazyRepeatedPtrField; // Concrete Arena enabled copy function used to copy messages instances. // This follows the `Arena::CopyConstruct` signature so that the compiler From a8d83db3c7a48779581b39978631caeefe4c2e58 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 17 Jul 2024 15:44:46 -0700 Subject: [PATCH 003/107] Add ::hpb::ExtensionNumber API for extension field constants. PiperOrigin-RevId: 653385818 --- hpb/hpb.h | 18 ++++++++++++++++++ hpb_generator/tests/BUILD | 3 ++- hpb_generator/tests/test_generated.cc | 12 +++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/hpb/hpb.h b/hpb/hpb.h index 8de8e249e8..7ae1b605e8 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -8,11 +8,13 @@ #ifndef PROTOBUF_HPB_HPB_H_ #define PROTOBUF_HPB_HPB_H_ +#include #include #include #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "upb/base/status.hpp" #include "upb/mem/arena.hpp" #include "upb/message/copy.h" @@ -148,6 +150,10 @@ struct PrivateAccess { static auto CreateMessage(upb_Arena* arena) { return typename T::Proxy(upb_Message_New(T::minitable(), arena), arena); } + template + static constexpr uint32_t GetExtensionNumber(const ExtensionId& id) { + return id.number(); + } }; template @@ -200,6 +206,12 @@ class ExtensionIdentifier : public ExtensionMiniTableProvider { constexpr explicit ExtensionIdentifier( const upb_MiniTableExtension* mini_table_ext) : ExtensionMiniTableProvider(mini_table_ext) {} + + private: + constexpr uint32_t number() const { + return upb_MiniTableExtension_Number(mini_table_ext()); + } + friend class PrivateAccess; }; template @@ -574,6 +586,12 @@ absl::StatusOr Serialize(Ptr message, upb::Arena& arena, ::protos::internal::GetMiniTable(message), arena.ptr(), options); } +template +constexpr uint32_t ExtensionNumber( + internal::ExtensionIdentifier id) { + return internal::PrivateAccess::GetExtensionNumber(id); +} + } // namespace protos #endif // PROTOBUF_HPB_HPB_H_ diff --git a/hpb_generator/tests/BUILD b/hpb_generator/tests/BUILD index 0109c6dd06..111944bebd 100644 --- a/hpb_generator/tests/BUILD +++ b/hpb_generator/tests/BUILD @@ -156,8 +156,9 @@ # "@com_google_absl//absl/strings", # "//hpb:requires", # "//hpb", -# "//upb:mem", # "//hpb:repeated_field", +# "//protos", +# "//upb:mem", # ], # ) # end:google_only diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index 12068b0234..c007d31d7c 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -20,8 +20,10 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/hpb/tests/child_model.upb.proto.h" #include "google/protobuf/compiler/hpb/tests/no_package.upb.proto.h" +#include "google/protobuf/compiler/hpb/tests/test_extension.upb.proto.h" #include "google/protobuf/compiler/hpb/tests/test_model.upb.proto.h" #include "google/protobuf/hpb/hpb.h" +#include "google/protobuf/hpb/repeated_field.h" #include "google/protobuf/hpb/requires.h" #include "upb/mem/arena.h" #include "upb/mem/arena.hpp" @@ -154,11 +156,11 @@ TEST(CppGeneratedCode, ScalarInt64) { EXPECT_EQ(testModel.optional_int64(), 0); EXPECT_FALSE(testModel.has_optional_int64()); // Set value. - testModel.set_optional_int64(0xFF00CCDDA0001000); + testModel.set_optional_int64(static_cast(0xFF00CCDDA0001000)); EXPECT_TRUE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0xFF00CCDDA0001000); // Change value. - testModel.set_optional_int64(0xFF00CCDD70002000); + testModel.set_optional_int64(static_cast(0xFF00CCDD70002000)); EXPECT_TRUE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0xFF00CCDD70002000); // Clear value. @@ -166,7 +168,7 @@ TEST(CppGeneratedCode, ScalarInt64) { EXPECT_FALSE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0); // Set after clear. - testModel.set_optional_int64(0xFF00CCDDA0001000); + testModel.set_optional_int64(static_cast(0xFF00CCDDA0001000)); EXPECT_TRUE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0xFF00CCDDA0001000); } @@ -1239,6 +1241,10 @@ TEST(CppGeneratedCode, FieldNumberConstants) { EXPECT_EQ(225, TestModel::kChildMapFieldNumber); } +TEST(CppGeneratedCode, ExtensionFieldNumberConstant) { + EXPECT_EQ(12003, ::protos::ExtensionNumber(ThemeExtension::theme_extension)); +} + TEST(CppGeneratedCode, ClearConstMessageShouldFailForConstChild) { TestModel model; EXPECT_FALSE(CanCallClearMessage()); From 90250161d49ff7e81c6d98f929aa81b581fa91bd Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 16:38:24 -0700 Subject: [PATCH 004/107] Remove unnecessarily suppressed warnings: rawtypes PiperOrigin-RevId: 653400633 --- .../src/main/java/com/google/protobuf/GeneratedMessage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java index e8a7cc8f97..b8610a8ca8 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -843,7 +843,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial * the generated API only allows us to access it as a map. This method returns the underlying * map field directly and thus enables us to access the map field as a list. */ - @SuppressWarnings({"unused", "rawtypes"}) + @SuppressWarnings("unused") protected MapFieldReflectionAccessor internalGetMapFieldReflection(int fieldNumber) { return internalGetMapField(fieldNumber); } @@ -858,7 +858,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial } /** Like {@link #internalGetMapFieldReflection} but return a mutable version. */ - @SuppressWarnings({"unused", "rawtypes"}) + @SuppressWarnings("unused") protected MapFieldReflectionAccessor internalGetMutableMapFieldReflection(int fieldNumber) { return internalGetMutableMapField(fieldNumber); } From af04a770aed1a2d1cde3a16584bbe0173cb28e58 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 16:55:29 -0700 Subject: [PATCH 005/107] Remove unused constructor parameter RepeatedFieldAccessor.descriptor PiperOrigin-RevId: 653405094 --- .../main/java/com/google/protobuf/GeneratedMessage.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java index b8610a8ca8..d22b49820a 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -2054,8 +2054,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial new RepeatedEnumFieldAccessor( field, camelCaseNames[i], messageClass, builderClass); } else { - fields[i] = - new RepeatedFieldAccessor(field, camelCaseNames[i], messageClass, builderClass); + fields[i] = new RepeatedFieldAccessor(camelCaseNames[i], messageClass, builderClass); } } else { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { @@ -2611,7 +2610,6 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial protected final MethodInvoker invoker; RepeatedFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, final Class messageClass, final Class> builderClass) { @@ -2916,7 +2914,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial final String camelCaseName, final Class messageClass, final Class> builderClass) { - super(descriptor, camelCaseName, messageClass, builderClass); + super(camelCaseName, messageClass, builderClass); enumDescriptor = descriptor.getEnumType(); @@ -3112,7 +3110,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial final String camelCaseName, final Class messageClass, final Class> builderClass) { - super(descriptor, camelCaseName, messageClass, builderClass); + super(camelCaseName, messageClass, builderClass); newBuilderMethod = getMethodOrDie(type, "newBuilder"); getBuilderMethodBuilder = From bc03650b3eefab6aea5b1b7019bffef8a74790c4 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 17:15:32 -0700 Subject: [PATCH 006/107] Add more tests for UnknownFieldSet Builder reusability Split up from testFieldBuildersAreReusable This is to build confidence I'm not breaking things with my upcoming changes. PiperOrigin-RevId: 653410161 --- .../google/protobuf/UnknownFieldSetTest.java | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java index f34a448c51..58ac736d4a 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java @@ -80,7 +80,7 @@ public class UnknownFieldSetTest { // ================================================================= @Test - public void testFieldBuildersAreReusable() { + public void testFixed32FieldBuildersAreReusable() { UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); fieldBuilder.addFixed32(10); UnknownFieldSet.Field first = fieldBuilder.build(); @@ -92,6 +92,64 @@ public class UnknownFieldSetTest { assertThat(first).isNotEqualTo(third); } + @Test + public void testFixed64FieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addFixed64(10); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addFixed64(11); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + + @Test + public void testVarintFieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addVarint(10); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addVarint(11); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + + @Test + public void testLengthDelimitedFieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addLengthDelimited(ByteString.copyFromUtf8("foo")); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addLengthDelimited(ByteString.copyFromUtf8("bar")); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + + @Test + public void testGroupFieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addGroup( + UnknownFieldSet.newBuilder() + .addField(10, UnknownFieldSet.Field.newBuilder().addVarint(10).build()) + .build()); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addGroup( + UnknownFieldSet.newBuilder() + .addField(11, UnknownFieldSet.Field.newBuilder().addVarint(11).build()) + .build()); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + @Test public void testClone() { UnknownFieldSet.Builder unknownSetBuilder = UnknownFieldSet.newBuilder(); From ed54faf7368a440a4711ef62cec0a26e82f15d5d Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 18:38:26 -0700 Subject: [PATCH 007/107] Avoid allocations in FieldSet.setField - Pre-size the ArrayList - Avoid allocating iterator Fix some rawtypes warnings too. PiperOrigin-RevId: 653430395 --- .../src/main/java/com/google/protobuf/FieldSet.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/FieldSet.java b/java/core/src/main/java/com/google/protobuf/FieldSet.java index f536be2640..9ddf548865 100644 --- a/java/core/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/FieldSet.java @@ -264,7 +264,8 @@ final class FieldSet> { /** * Useful for implementing {@link Message.Builder#setField(Descriptors.FieldDescriptor,Object)}. */ - @SuppressWarnings({"unchecked", "rawtypes"}) + // Avoid iterator allocation. + @SuppressWarnings({"ForeachList", "ForeachListWithUserVar"}) public void setField(final T descriptor, Object value) { if (descriptor.isRepeated()) { if (!(value instanceof List)) { @@ -274,10 +275,14 @@ final class FieldSet> { // Wrap the contents in a new list so that the caller cannot change // the list's contents after setting it. - final List newList = new ArrayList<>(); - newList.addAll((List) value); - for (final Object element : newList) { + List list = (List) value; + int listSize = list.size(); + // Avoid extra allocations: no iterator, no intermediate array copy. + final List newList = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + Object element = list.get(i); verifyType(descriptor, element); + newList.add(element); } value = newList; } else { From 5b45a655678266fc2e9d7bb50de1963d362b1a5d Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 19:16:06 -0700 Subject: [PATCH 008/107] Simplify some compareTo operations I believe they have intrinsics even for the long-form Integer.compareTo(Integer.valueOf(a), Integer.valueOf(b)) format which avoids the intermediate allocation. So this probably won't make things faster, just makes the code a little cleaner. Integer.compare was added in Java 1.7 which seems safe. Added to Android in SDK 19, which is less than the 21 minSDK supported by protobuf: https://github.com/protocolbuffers/protobuf/commit/303239d74d07daf49ecdba6e23e41c8fe5a9b41e PiperOrigin-RevId: 653438420 --- java/core/src/main/java/com/google/protobuf/ByteString.java | 5 ++--- java/core/src/main/java/com/google/protobuf/Descriptors.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 7b2455f12d..801347e0fd 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -287,13 +287,12 @@ public abstract class ByteString implements Iterable, Serializable { while (formerBytes.hasNext() && latterBytes.hasNext()) { int result = - Integer.valueOf(toInt(formerBytes.nextByte())) - .compareTo(toInt(latterBytes.nextByte())); + Integer.compare(toInt(formerBytes.nextByte()), toInt(latterBytes.nextByte())); if (result != 0) { return result; } } - return Integer.valueOf(former.size()).compareTo(Integer.valueOf(latter.size())); + return Integer.compare(former.size(), latter.size()); } }; diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 569fa26e0e..bade681e1d 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -2343,7 +2343,7 @@ public final class Descriptors { new Comparator() { @Override public int compare(EnumValueDescriptor o1, EnumValueDescriptor o2) { - return Integer.valueOf(o1.getNumber()).compareTo(o2.getNumber()); + return Integer.compare(o1.getNumber(), o2.getNumber()); } }; From cb323f4359a3b2d2d2612300429ec14a4a3478dd Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 19:50:53 -0700 Subject: [PATCH 009/107] Avoid unboxing boxed primitives that we're just going to re-box back up to call compareTo with. This may save some alloactions if java's intrinsics aren't smart enough to avoid the roundtrip. But most JVMs probably have smart enough intrinsics, so this is probably not going to speed things up, just make the code look nicer. PiperOrigin-RevId: 653445330 --- java/core/src/main/java/com/google/protobuf/TextFormat.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index ddd51d0675..8771403b1f 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -449,11 +449,11 @@ public final class TextFormat { } switch (fieldType) { case BOOLEAN: - return Boolean.valueOf((boolean) getKey()).compareTo((boolean) b.getKey()); + return ((Boolean) getKey()).compareTo((Boolean) b.getKey()); case LONG: - return Long.valueOf((long) getKey()).compareTo((long) b.getKey()); + return ((Long) getKey()).compareTo((Long) b.getKey()); case INT: - return Integer.valueOf((int) getKey()).compareTo((int) b.getKey()); + return ((Integer) getKey()).compareTo((Integer) b.getKey()); case STRING: String aString = (String) getKey(); String bString = (String) b.getKey(); From be4f149abd68ab6f78b30976118fac5da6dad976 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 17 Jul 2024 20:08:14 -0700 Subject: [PATCH 010/107] Extract MAX_UINT32 BigInteger into a constant, so we aren't constructing it all the time PiperOrigin-RevId: 653450622 --- .../src/main/java/com/google/protobuf/util/JsonFormat.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 5092588132..97bdbb0b70 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -1790,7 +1790,7 @@ public class JsonFormat { try { BigDecimal decimalValue = new BigDecimal(json.getAsString()); BigInteger value = decimalValue.toBigIntegerExact(); - if (value.signum() < 0 || value.compareTo(new BigInteger("FFFFFFFF", 16)) > 0) { + if (value.signum() < 0 || value.compareTo(MAX_UINT32) > 0) { throw new InvalidProtocolBufferException("Out of range uint32 value: " + json); } return value.intValue(); @@ -1802,6 +1802,7 @@ public class JsonFormat { } } + private static final BigInteger MAX_UINT32 = new BigInteger("FFFFFFFF", 16); private static final BigInteger MAX_UINT64 = new BigInteger("FFFFFFFFFFFFFFFF", 16); private long parseUint64(JsonElement json) throws InvalidProtocolBufferException { From 7c5dd9ec6476a63b5127e49e1c47135affe45ed3 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 17 Jul 2024 23:06:29 -0700 Subject: [PATCH 011/107] Remove the second 'unset' generic argument from Optional This is no longer needed with our new design. PiperOrigin-RevId: 653488903 --- rust/optional.rs | 8 +++----- rust/test/shared/accessors_test.rs | 6 ++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/rust/optional.rs b/rust/optional.rs index 1f5cb9ad46..221f4d9ed2 100644 --- a/rust/optional.rs +++ b/rust/optional.rs @@ -23,14 +23,14 @@ use std::ptr; /// /// Two `Optional`s are equal if they match both presence and the field values. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Optional { +pub enum Optional { /// The field is set; it is present in the serialized message. /// /// - For an `_opt()` accessor, this contains a `View`. /// - For a `_mut()` accessor, this contains a [`PresentField`] that can be /// used to access the current value, convert to [`Mut`], clear presence, /// or set a new value. - Set(SetVal), + Set(T), /// The field is unset; it is absent in the serialized message. /// @@ -38,7 +38,7 @@ pub enum Optional { /// the default value. /// - For a `_mut()` accessor, this contains an [`AbsentField`] that can be /// used to access the default or set a new value. - Unset(UnsetVal), + Unset(T), } impl Optional { @@ -53,9 +53,7 @@ impl Optional { pub fn new(val: T, is_set: bool) -> Self { if is_set { Optional::Set(val) } else { Optional::Unset(val) } } -} -impl Optional { /// Converts into an `Option` of the set value, ignoring any unset value. pub fn into_option(self) -> Option { if let Optional::Set(x) = self { Some(x) } else { None } diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index f108c889c1..c0365b1d9e 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -609,10 +609,8 @@ fn test_singular_msg_field() { #[test] fn test_message_opt() { let msg = TestAllTypes::new(); - let opt: Optional< - unittest_rust_proto::test_all_types::NestedMessageView<'_>, - unittest_rust_proto::test_all_types::NestedMessageView<'_>, - > = msg.optional_nested_message_opt(); + let opt: Optional> = + msg.optional_nested_message_opt(); assert_that!(opt.is_set(), eq(false)); assert_that!(opt.into_inner().bb(), eq(0)); } From 3ff2cf05f341a2ebe9e4e79e59717f85cfd5d601 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 02:08:27 -0700 Subject: [PATCH 012/107] Move proto_library from Bazel repository Use paths.is_normalized and paths.is_absolute from bazel skylib. Upgrade skylib to latest version that has the implementation. Use copybara for the differences in STRIC_DEPS_FLAG_TEMPLATE. Implement native_bool_flag to read native flags and use them in proto_library. Those are implemented in such a way, that they can be replaced in place with starlark bool_flag targets. Implement version check for PackageSpecificationInfo. It's only available after Bazel 6.4.0. Tests will be submitted in separate PRs. PiperOrigin-RevId: 653532601 --- MODULE.bazel | 2 +- bazel/private/BUILD.bazel | 24 ++ bazel/private/native_bool_flag.bzl | 35 +++ bazel/private/proto_library_rule.bzl | 357 +++++++++++++++++++++++++++ bazel/proto_library.bzl | 21 +- protobuf_deps.bzl | 6 +- 6 files changed, 439 insertions(+), 6 deletions(-) create mode 100644 bazel/private/native_bool_flag.bzl create mode 100644 bazel/private/proto_library_rule.bzl diff --git a/MODULE.bazel b/MODULE.bazel index b9f9fbf24d..fcf2eeaa95 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ module( # https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution # Thus the highest version in their module graph is resolved. bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl") -bazel_dep(name = "bazel_skylib", version = "1.4.1") +bazel_dep(name = "bazel_skylib", version = "1.7.0") bazel_dep(name = "jsoncpp", version = "1.9.5") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "rules_fuzzing", version = "0.5.2") diff --git a/bazel/private/BUILD.bazel b/bazel/private/BUILD.bazel index 1ff703b53d..407c325916 100644 --- a/bazel/private/BUILD.bazel +++ b/bazel/private/BUILD.bazel @@ -6,6 +6,7 @@ # https://developers.google.com/open-source/licenses/bsd load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//bazel/private:native_bool_flag.bzl", "native_bool_flag") licenses(["notice"]) @@ -52,3 +53,26 @@ bzl_library( "//bazel/common:proto_lang_toolchain_info_bzl", ], ) + +native_bool_flag( + name = "experimental_proto_descriptor_sets_include_source_info", + flag = "experimental_proto_descriptor_sets_include_source_info", + match_value = "true", + visibility = ["//visibility:public"], +) + +native_bool_flag( + name = "strict_proto_deps", + flag = "strict_proto_deps", + match_value = "off", + result = False, + visibility = ["//visibility:public"], +) + +native_bool_flag( + name = "strict_public_imports", + flag = "strict_public_imports", + match_value = "off", + result = False, + visibility = ["//visibility:public"], +) diff --git a/bazel/private/native_bool_flag.bzl b/bazel/private/native_bool_flag.bzl new file mode 100644 index 0000000000..960fbed512 --- /dev/null +++ b/bazel/private/native_bool_flag.bzl @@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +A helper rule that reads a native boolean flag. +""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") + +def _impl(ctx): + return [BuildSettingInfo(value = ctx.attr.value)] + +_native_bool_flag_rule = rule( + implementation = _impl, + attrs = {"value": attr.bool()}, +) + +def native_bool_flag(*, name, flag, match_value = "true", result = True, **kwargs): + _native_bool_flag_rule( + name = name, + value = select({ + name + "_setting": result, + "//conditions:default": not result, + }), + **kwargs + ) + + native.config_setting( + name = name + "_setting", + values = {flag: match_value}, + visibility = ["//visibility:private"], + ) diff --git a/bazel/private/proto_library_rule.bzl b/bazel/private/proto_library_rule.bzl new file mode 100644 index 0000000000..b02f691f45 --- /dev/null +++ b/bazel/private/proto_library_rule.bzl @@ -0,0 +1,357 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +Implementation of proto_library rule. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/common:proto_common.bzl", "proto_common") +load("//bazel/common:proto_info.bzl", "ProtoInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") + +STRICT_DEPS_FLAG_TEMPLATE = ( + # + "--direct_dependencies_violation_msg=" + + "%%s is imported, but %s doesn't directly depend on a proto_library that 'srcs' it." +) + +def _check_srcs_package(target_package, srcs): + """Check that .proto files in sources are from the same package. + + This is done to avoid clashes with the generated sources.""" + + #TODO: this does not work with filegroups that contain files that are not in the package + for src in srcs: + if target_package != src.label.package: + fail("Proto source with label '%s' must be in same package as consuming rule." % src.label) + +def _get_import_prefix(ctx): + """Gets and verifies import_prefix attribute if it is declared.""" + + import_prefix = ctx.attr.import_prefix + + if not paths.is_normalized(import_prefix): + fail("should be normalized (without uplevel references or '.' path segments)", attr = "import_prefix") + if paths.is_absolute(import_prefix): + fail("should be a relative path", attr = "import_prefix") + + return import_prefix + +def _get_strip_import_prefix(ctx): + """Gets and verifies strip_import_prefix.""" + + strip_import_prefix = ctx.attr.strip_import_prefix + + if not paths.is_normalized(strip_import_prefix): + fail("should be normalized (without uplevel references or '.' path segments)", attr = "strip_import_prefix") + + if paths.is_absolute(strip_import_prefix): + strip_import_prefix = strip_import_prefix[1:] + else: # Relative to current package + strip_import_prefix = _join(ctx.label.package, strip_import_prefix) + + return strip_import_prefix.removesuffix("/") + +def _proto_library_impl(ctx): + # Verifies attributes. + _check_srcs_package(ctx.label.package, ctx.attr.srcs) + srcs = ctx.files.srcs + deps = [dep[ProtoInfo] for dep in ctx.attr.deps] + exports = [dep[ProtoInfo] for dep in ctx.attr.exports] + import_prefix = _get_import_prefix(ctx) + strip_import_prefix = _get_strip_import_prefix(ctx) + check_for_reexport = deps + exports if not srcs else exports + _PackageSpecificationInfo = bazel_features.globals.PackageSpecificationInfo + for proto in check_for_reexport: + if getattr(proto, "allow_exports", None): + if not _PackageSpecificationInfo: + fail("Allowlist checks not supported before Bazel 6.4.0") + if not proto.allow_exports[_PackageSpecificationInfo].contains(ctx.label): + fail("proto_library '%s' can't be reexported in package '//%s'" % (proto.direct_descriptor_set.owner, ctx.label.package)) + + proto_path, virtual_srcs = _process_srcs(ctx, srcs, import_prefix, strip_import_prefix) + descriptor_set = ctx.actions.declare_file(ctx.label.name + "-descriptor-set.proto.bin") + proto_info = ProtoInfo( + srcs = virtual_srcs, + deps = deps, + descriptor_set = descriptor_set, + proto_path = proto_path, + workspace_root = ctx.label.workspace_root, + bin_dir = ctx.bin_dir.path, + allow_exports = ctx.attr.allow_exports, + ) + + _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set) + + # We assume that the proto sources will not have conflicting artifacts + # with the same root relative path + data_runfiles = ctx.runfiles( + files = [proto_info.direct_descriptor_set], + transitive_files = depset(transitive = [proto_info.transitive_sources]), + ) + return [ + proto_info, + DefaultInfo( + files = depset([proto_info.direct_descriptor_set]), + default_runfiles = ctx.runfiles(), # empty + data_runfiles = data_runfiles, + ), + ] + +def _process_srcs(ctx, srcs, import_prefix, strip_import_prefix): + """Returns proto_path and sources, optionally symlinking them to _virtual_imports. + + Returns: + (str, [File]) A pair of proto_path and virtual_sources. + """ + if import_prefix != "" or strip_import_prefix != "": + # Use virtual source roots + return _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix) + else: + # No virtual source roots + return "", srcs + +def _join(*path): + return "/".join([p for p in path if p != ""]) + +def _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix): + """Symlinks srcs to _virtual_imports. + + Returns: + A pair proto_path, directs_sources. + """ + virtual_imports = _join("_virtual_imports", ctx.label.name) + proto_path = _join(ctx.label.package, virtual_imports) + + if ctx.label.workspace_name == "": + full_strip_import_prefix = strip_import_prefix + else: + full_strip_import_prefix = _join("..", ctx.label.workspace_name, strip_import_prefix) + if full_strip_import_prefix: + full_strip_import_prefix += "/" + + virtual_srcs = [] + for src in srcs: + # Remove strip_import_prefix + if not src.short_path.startswith(full_strip_import_prefix): + fail(".proto file '%s' is not under the specified strip prefix '%s'" % + (src.short_path, full_strip_import_prefix)) + import_path = src.short_path[len(full_strip_import_prefix):] + + # Add import_prefix + virtual_src = ctx.actions.declare_file(_join(virtual_imports, import_prefix, import_path)) + ctx.actions.symlink( + output = virtual_src, + target_file = src, + progress_message = "Symlinking virtual .proto sources for %{label}", + ) + virtual_srcs.append(virtual_src) + return proto_path, virtual_srcs + +def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): + """Writes descriptor set.""" + if proto_info.direct_sources == []: + ctx.actions.write(descriptor_set, "") + return + + dependencies_descriptor_sets = depset(transitive = [dep.transitive_descriptor_sets for dep in deps]) + + args = ctx.actions.args() + + if ctx.attr._experimental_proto_descriptor_sets_include_source_info[BuildSettingInfo].value: + args.add("--include_source_info") + if hasattr(ctx.attr, "_retain_options") and ctx.attr._retain_options: + args.add("--retain_options") + + strict_deps = ctx.attr._strict_proto_deps[BuildSettingInfo].value + if strict_deps: + if proto_info.direct_sources: + strict_importable_sources = depset( + direct = proto_info._direct_proto_sources, + transitive = [dep._exported_sources for dep in deps], + ) + else: + strict_importable_sources = None + if strict_importable_sources: + args.add_joined( + "--direct_dependencies", + strict_importable_sources, + map_each = proto_common.get_import_path, + join_with = ":", + ) + # Example: `--direct_dependencies a.proto:b.proto` + + else: + # The proto compiler requires an empty list to turn on strict deps checking + args.add("--direct_dependencies=") + + # Set `-direct_dependencies_violation_msg=` + args.add(ctx.label, format = STRICT_DEPS_FLAG_TEMPLATE) + + strict_imports = ctx.attr._strict_public_imports[BuildSettingInfo].value + if strict_imports: + public_import_protos = depset(transitive = [export._exported_sources for export in exports]) + if not public_import_protos: + # This line is necessary to trigger the check. + args.add("--allowed_public_imports=") + else: + args.add_joined( + "--allowed_public_imports", + public_import_protos, + map_each = proto_common.get_import_path, + join_with = ":", + ) + if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: + toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN] + if not toolchain: + fail("Protocol compiler toolchain could not be resolved.") + proto_lang_toolchain_info = toolchain.proto + else: + proto_lang_toolchain_info = proto_common.ProtoLangToolchainInfo( + out_replacement_format_flag = "--descriptor_set_out=%s", + output_files = "single", + mnemonic = "GenProtoDescriptorSet", + progress_message = "Generating Descriptor Set proto_library %{label}", + proto_compiler = ctx.executable._proto_compiler, + protoc_opts = ctx.fragments.proto.experimental_protoc_opts, + plugin = None, + ) + + proto_common.compile( + ctx.actions, + proto_info, + proto_lang_toolchain_info, + generated_files = [descriptor_set], + additional_inputs = dependencies_descriptor_sets, + additional_args = args, + ) + +proto_library = rule( + _proto_library_impl, + # TODO: proto_common docs are missing + # TODO: ProtoInfo link doesn't work and docs are missing + doc = """ +

If using Bazel, please load the rule from +https://github.com/bazelbuild/rules_proto. + +

Use proto_library to define libraries of protocol buffers which +may be used from multiple languages. A proto_library may be listed +in the deps clause of supported rules, such as +java_proto_library. + +

When compiled on the command-line, a proto_library creates a file +named foo-descriptor-set.proto.bin, which is the descriptor set for +the messages the rule srcs. The file is a serialized +FileDescriptorSet, which is described in + +https://developers.google.com/protocol-buffers/docs/techniques#self-description. + +

It only contains information about the .proto files directly +mentioned by a proto_library rule; the collection of transitive +descriptor sets is available through the +[ProtoInfo].transitive_descriptor_sets Starlark provider. +See documentation in proto_info.bzl. + +

Recommended code organization: +

    +
  • One proto_library rule per .proto file. +
  • A file named foo.proto will be in a rule named foo_proto, + which is located in the same package. +
  • A [language]_proto_library that wraps a proto_library + named foo_proto should be called foo_[language]_proto, + and be located in the same package. +
""", + attrs = { + "srcs": attr.label_list( + allow_files = [".proto", ".protodevel"], + flags = ["DIRECT_COMPILE_TIME_INPUT"], + # TODO: Should .protodevel be advertised or deprecated? + doc = """ +The list of .proto and .protodevel files that are +processed to create the target. This is usually a non empty list. One usecase +where srcs can be empty is an alias-library. This is a +proto_library rule having one or more other proto_library in deps. +This pattern can be used to e.g. export a public api under a persistent name.""", + ), + "deps": attr.label_list( + providers = [ProtoInfo], + doc = """ +The list of other proto_library rules that the target depends upon. +A proto_library may only depend on other proto_library +targets. It may not depend on language-specific libraries.""", + ), + "exports": attr.label_list( + providers = [ProtoInfo], + doc = """ +List of proto_library targets that can be referenced via "import public" in the +proto source. +It's an error if you use "import public" but do not list the corresponding library +in the exports attribute. +Note that you have list the library both in deps and exports since not all +lang_proto_library implementations have been changed yet.""", + ), + "strip_import_prefix": attr.string( + default = "/", + doc = """ +The prefix to strip from the paths of the .proto files in this rule. + +

When set, .proto source files in the srcs attribute of this rule are +accessible at their path with this prefix cut off. + +

If it's a relative path (not starting with a slash), it's taken as a package-relative +one. If it's an absolute one, it's understood as a repository-relative path. + +

The prefix in the import_prefix attribute is added after this prefix is +stripped.""", + ), + "import_prefix": attr.string( + doc = """ +The prefix to add to the paths of the .proto files in this rule. + +

When set, the .proto source files in the srcs attribute of this rule are +accessible at is the value of this attribute prepended to their repository-relative path. + +

The prefix in the strip_import_prefix attribute is removed before this +prefix is added.""", + ), + "allow_exports": attr.label( + cfg = "exec", + providers = [bazel_features.globals.PackageSpecificationInfo] if bazel_features.globals.PackageSpecificationInfo else [], + doc = """ +An optional allowlist that prevents proto library to be reexported or used in +lang_proto_library that is not in one of the listed packages.""", + ), + "data": attr.label_list( + allow_files = True, + flags = ["SKIP_CONSTRAINTS_OVERRIDE"], + ), + # buildifier: disable=attr-license (calling attr.license()) + "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), + "_experimental_proto_descriptor_sets_include_source_info": attr.label( + default = "//bazel/private:experimental_proto_descriptor_sets_include_source_info", + ), + "_strict_proto_deps": attr.label( + default = + "//bazel/private:strict_proto_deps", + ), + "_strict_public_imports": attr.label( + default = "//bazel/private:strict_public_imports", + ), + } | toolchains.if_legacy_toolchain({ + "_proto_compiler": attr.label( + cfg = "exec", + executable = True, + allow_files = True, + default = configuration_field("proto", "proto_compiler"), + ), + }), # buildifier: disable=attr-licenses (attribute called licenses) + fragments = ["proto"], + provides = [ProtoInfo], + toolchains = toolchains.use_toolchain(toolchains.PROTO_TOOLCHAIN), +) diff --git a/bazel/proto_library.bzl b/bazel/proto_library.bzl index 3006e3c0c2..f0fece1553 100644 --- a/bazel/proto_library.bzl +++ b/bazel/proto_library.bzl @@ -1,3 +1,20 @@ -"""proto_library rule""" +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +Macro of proto_library rule. +""" -proto_library = native.proto_library +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/private:proto_library_rule.bzl", _proto_library = "proto_library") + +def proto_library(**kwattrs): + # This condition causes Starlark rules to be used only on Bazel >=7.0.0 + if bazel_features.proto.starlark_proto_info: + _proto_library(**kwattrs) + else: + # On older Bazel versions keep using native rules, so that mismatch in ProtoInfo doesn't happen + native.proto_library(**kwattrs) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9dcf0c0428..eea6072696 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -51,11 +51,11 @@ def protobuf_deps(): if not native.existing_rule("bazel_skylib"): http_archive( name = "bazel_skylib", + sha256 = "d00f1389ee20b60018e92644e0948e16e350a7707219e7a390fb0a99b6ec9262", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.7.0/bazel-skylib-1.7.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.7.0/bazel-skylib-1.7.0.tar.gz", ], - sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", ) if not native.existing_rule("com_google_absl"): From cc8b3483a5584b3301e3d43d17eb59704857ffaa Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 07:41:01 -0700 Subject: [PATCH 013/107] Internal change PiperOrigin-RevId: 653615736 --- .../core/src/main/java/com/google/protobuf/ArrayDecoders.java | 3 +-- .../com/google/protobuf/InvalidProtocolBufferException.java | 2 +- .../core/src/main/java/com/google/protobuf/MessageSchema.java | 3 +++ .../src/main/java/com/google/protobuf/MessageSetSchema.java | 1 + .../src/main/java/com/google/protobuf/UnknownFieldSchema.java | 3 +-- java/lite/src/test/java/com/google/protobuf/LiteTest.java | 3 +++ src/google/protobuf/unittest_lite.proto | 4 ++++ 7 files changed, 14 insertions(+), 5 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java b/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java index f3241de509..9bf1439626 100644 --- a/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java +++ b/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java @@ -24,8 +24,7 @@ import java.io.IOException; @CheckReturnValue final class ArrayDecoders { - private ArrayDecoders() { - } + private ArrayDecoders() {} /** * A helper used to return multiple values in a Java function. Java doesn't natively support diff --git a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java index 5d10e48884..dbcb9e899d 100644 --- a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java +++ b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java @@ -132,7 +132,7 @@ public class InvalidProtocolBufferException extends IOException { static InvalidProtocolBufferException recursionLimitExceeded() { return new InvalidProtocolBufferException( "Protocol message had too many levels of nesting. May be malicious. " - + "Use CodedInputStream.setRecursionLimit() to increase the depth limit."); + + "Use setRecursionLimit() to increase the recursion depth limit."); } static InvalidProtocolBufferException sizeLimitExceeded() { diff --git a/java/core/src/main/java/com/google/protobuf/MessageSchema.java b/java/core/src/main/java/com/google/protobuf/MessageSchema.java index de3890f702..f8f79fcdf8 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageSchema.java +++ b/java/core/src/main/java/com/google/protobuf/MessageSchema.java @@ -3006,6 +3006,7 @@ final class MessageSchema implements Schema { unknownFields = unknownFieldSchema.getBuilderFromMessage(message); } // Unknown field. + if (unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader)) { continue; } @@ -3381,6 +3382,7 @@ final class MessageSchema implements Schema { if (unknownFields == null) { unknownFields = unknownFieldSchema.getBuilderFromMessage(message); } + if (!unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader)) { return; } @@ -3397,6 +3399,7 @@ final class MessageSchema implements Schema { if (unknownFields == null) { unknownFields = unknownFieldSchema.getBuilderFromMessage(message); } + if (!unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader)) { return; } diff --git a/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java b/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java index eec3acd35c..a17037e8ef 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java +++ b/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java @@ -278,6 +278,7 @@ final class MessageSetSchema implements Schema { reader, extension, extensionRegistry, extensions); return true; } else { + return unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader); } } else { diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java index c4ec645bf7..a43bc2a947 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java @@ -55,7 +55,6 @@ abstract class UnknownFieldSchema { /** Marks unknown fields as immutable. */ abstract void makeImmutable(Object message); - /** Merges one field into the unknown fields. */ final boolean mergeOneFieldFrom(B unknownFields, Reader reader) throws IOException { int tag = reader.getTag(); int fieldNumber = WireFormat.getTagFieldNumber(tag); @@ -88,7 +87,7 @@ abstract class UnknownFieldSchema { } } - final void mergeFrom(B unknownFields, Reader reader) throws IOException { + private final void mergeFrom(B unknownFields, Reader reader) throws IOException { while (true) { if (reader.getFieldNumber() == Reader.READ_DONE || !mergeOneFieldFrom(unknownFields, reader)) { diff --git a/java/lite/src/test/java/com/google/protobuf/LiteTest.java b/java/lite/src/test/java/com/google/protobuf/LiteTest.java index 0c7b8b535b..411bc63f08 100644 --- a/java/lite/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/lite/src/test/java/com/google/protobuf/LiteTest.java @@ -10,12 +10,14 @@ package com.google.protobuf; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static java.util.Collections.singletonList; +import static org.junit.Assert.assertThrows; import com.google.protobuf.FieldPresenceTestProto.TestAllTypes; import com.google.protobuf.UnittestImportLite.ImportEnumLite; import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; import com.google.protobuf.UnittestLite.ForeignEnumLite; import com.google.protobuf.UnittestLite.ForeignMessageLite; +import com.google.protobuf.UnittestLite.RecursiveGroup; import com.google.protobuf.UnittestLite.RecursiveMessage; import com.google.protobuf.UnittestLite.TestAllExtensionsLite; import com.google.protobuf.UnittestLite.TestAllTypesLite; @@ -50,6 +52,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/google/protobuf/unittest_lite.proto b/src/google/protobuf/unittest_lite.proto index b3fcfa431c..4bc78c4de6 100644 --- a/src/google/protobuf/unittest_lite.proto +++ b/src/google/protobuf/unittest_lite.proto @@ -625,3 +625,7 @@ message RecursiveMessage { RecursiveMessage recurse = 1; bytes payload = 2; } + +message RecursiveGroup { + RecursiveGroup recurse = 1 [features.message_encoding = DELIMITED]; +} From 0b4173e3edd5970fea27759d62e573ea7561d400 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 07:52:43 -0700 Subject: [PATCH 014/107] Upgrade googetest to 1.15.0 PiperOrigin-RevId: 653618528 --- WORKSPACE | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 5ca0070466..74a96c7455 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -41,10 +41,10 @@ http_archive( http_archive( name = "com_google_googletest", - sha256 = "730215d76eace9dd49bf74ce044e8daa065d175f1ac891cc1d6bb184ef94e565", - strip_prefix = "googletest-f53219cdcb7b084ef57414efea92ee5b71989558", + sha256 = "7315acb6bf10e99f332c8a43f00d5fbb1ee6ca48c52f6b936991b216c586aaad", + strip_prefix = "googletest-1.15.0", urls = [ - "https://github.com/google/googletest/archive/f53219cdcb7b084ef57414efea92ee5b71989558.tar.gz" # 2023-03-16 + "https://github.com/google/googletest/releases/download/v1.15.0/googletest-1.15.0.tar.gz" # 2024-07-15 ], ) From a6a0680e95f667e216fad17bee2f968f7e0f1bd7 Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Thu, 18 Jul 2024 08:28:06 -0700 Subject: [PATCH 015/107] allow __ for inferred submessage types and implement array literal parsing in proto!. PiperOrigin-RevId: 653628636 --- rust/proto_macro.rs | 226 ++++++++++++++++++++++++--- rust/shared.rs | 7 + rust/test/shared/proto_macro_test.rs | 51 ++++++ 3 files changed, 258 insertions(+), 26 deletions(-) diff --git a/rust/proto_macro.rs b/rust/proto_macro.rs index a70afffa62..b3f937d605 100644 --- a/rust/proto_macro.rs +++ b/rust/proto_macro.rs @@ -40,6 +40,8 @@ macro_rules! proto { #[macro_export(local_inner_macros)] #[doc(hidden)] macro_rules! proto_internal { + // @merge rules are used to find a trailing ..expr on the message and call merge_from on it + // before the fields of the message are set. (@merge $msg:ident $ident:ident : $expr:expr, $($rest:tt)*) => { proto_internal!(@merge $msg $($rest)*); }; @@ -51,52 +53,224 @@ macro_rules! proto_internal { $msg.merge_from($expr); }; - // nested message, - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $field:ident : $($value:tt)* }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : $($msgtype)::+ { $field : $($value)* }); + // @msg rules are used to set the fields of the message. There is a lot of duplication here + // because we need to parse the message type using a :: separated list of identifiers. + // nested message and trailing fields + (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@msg $msg $submsg : $($msgtype)::+ { $($value)* }); proto_internal!(@msg $msg $($rest)*); }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $field:ident : $($value:tt)* }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : ::$($msgtype)::+ { $field : $($value)* }); + // nested message with leading :: on type and trailing fields + (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@msg $msg $submsg : ::$($msgtype)::+ { $($value)* }); proto_internal!(@msg $msg $($rest)*); }; + // nested message using __ + (@msg $msg:ident $submsg:ident : __ { $($value:tt)* }) => { + { + let mut $msg = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + } + }; // nested message - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $field:ident : $($value:tt)* }) => { + (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $($value:tt)* }) => { { let mut $msg: <$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); - proto_internal!(@merge $msg $field : $($value)*); - proto_internal!(@msg $msg $field : $($value)*); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); } }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $field:ident : $($value:tt)* }) => { + // nested message with leading :: + (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $($value:tt)* }) => { { let mut $msg: <::$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); - proto_internal!(@merge $msg $field : $($value)*); - proto_internal!(@msg $msg $field : $($value)*); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); } }; - // empty nested message, - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : $($msgtype)::+ { }); - proto_internal!(@msg $msg $($rest)*); - }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : ::$($msgtype)::+ { }); + // field with array literal and trailing fields + (@msg $msg:ident $ident:ident : [$($elems:tt)*], $($rest:tt)*) => { + proto_internal!(@msg $msg $ident : [$($elems)*]); proto_internal!(@msg $msg $($rest)*); }; - - // empty nested message - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { }) => { + // field with array literal, calls out to @array to look for nested messages + (@msg $msg:ident $ident:ident : [$($elems:tt)*]) => { { - let mut $msg: <$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + let _repeated = $crate::__internal::paste!($msg.[<$ident>]()); + let elems = proto_internal!(@array $msg _repeated [] $($elems)*); + $crate::__internal::paste!($msg.[](elems.into_iter())); } }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { }) => { - { - let mut $msg: <::$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); - } + + // @array searches through an array literal for nested messages. + // If a message is found then we recursively call the macro on it to set the fields. + // This will create an array literal of owned messages to be used while setting the field. + // For primitive types they should just be passed through as an $expr. + // The array literal is constructed recursively, so the [] case has to be handled separately so + // that we can properly insert commas. This leads to a lot of duplication. + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [$($vals:expr),+] __ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + $($vals),+ , + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + + // Message nested in [] literal + (@array $msg:ident $repeated:ident [$($vals:expr),+] __ { $($value:tt)* }) => { + [ + $($vals),+ , + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [] __ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + + // Message nested in [] literal + (@array $msg:ident $repeated:ident [] __ { $($value:tt)* }) => { + [ + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + // End of __ repeated, now we need to handle named types + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [$($vals:expr),+] $($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + $($vals),+ , + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // Message nested in [] literal with leading :: on type and trailing array items + (@array $msg:ident $repeated:ident [$($vals:expr),+] ::$($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + $($vals),+ , + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // Message nested in [] literal + (@array $msg:ident $repeated:ident [$($vals:expr),+] $($msgtype:ident)::+ { $($value:tt)* }) => { + [ + $($vals),+ , + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + // Message nested in [] literal with leading :: on type + (@array $msg:ident $repeated:ident [$($vals:expr),+] ::$($msgtype:ident)::+ { $($value:tt)* }) => { + [ + $($vals),+ , + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [] $($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // with leading :: + (@array $msg:ident $repeated:ident [] ::$($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // Message nested in [] literal + (@array $msg:ident $repeated:ident [] $($msgtype:ident)::+ { $($value:tt)* }) => { + [ + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + (@array $msg:ident $repeated:ident [] ::$($msgtype:ident)::+ { $($value:tt)* }) => { + [ + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + (@array $msg:ident $repeated:ident [$($vals:expr),+] $expr:expr, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [$($vals),+, $expr] $($rest)*) + }; + (@array $msg:ident $repeated:ident [$($vals:expr),+] $expr:expr) => { + [$($vals),+, $expr] + }; + (@array $msg:ident $repeated:ident [] $expr:expr, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [$expr] $($rest)*) + }; + (@array $msg:ident $repeated:ident [] $expr:expr) => { + [$expr] + }; + (@array $msg:ident $repeated:ident []) => { + [] }; // field: expr, diff --git a/rust/shared.rs b/rust/shared.rs index 403fcb0ea5..4494ee7019 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -91,3 +91,10 @@ impl fmt::Display for SerializeError { write!(f, "Couldn't serialize proto into bytes (depth too deep or missing required fields)") } } + +pub fn get_repeated_default_value( + _: __internal::Private, + _: repeated::RepeatedView<'_, T>, +) -> T { + Default::default() +} diff --git a/rust/test/shared/proto_macro_test.rs b/rust/test/shared/proto_macro_test.rs index 07c6f6bb80..876a5a47f4 100644 --- a/rust/test/shared/proto_macro_test.rs +++ b/rust/test/shared/proto_macro_test.rs @@ -94,6 +94,14 @@ fn single_nested_message() { }); assert_that!(msg.optional_nested_message().bb(), eq(42)); + // field above and below it + let msg = proto!(TestAllTypes { + optional_int32: 1, + optional_nested_message: __ { bb: 42 }, + optional_int64: 2 + }); + assert_that!(msg.optional_nested_message().bb(), eq(42)); + // test empty initializer let msg = proto!(TestAllTypes {}); assert_that!(msg.has_optional_nested_message(), eq(false)); @@ -104,6 +112,14 @@ fn single_nested_message() { optional_nested_message: unittest_rust_proto::test_all_types::NestedMessage {} }); assert_that!(msg.has_optional_nested_message(), eq(true)); + + let msg = proto!(::unittest_rust_proto::TestAllTypes { + optional_nested_message: ::unittest_rust_proto::test_all_types::NestedMessage {} + }); + assert_that!(msg.has_optional_nested_message(), eq(true)); + + let msg = proto!(::unittest_rust_proto::TestAllTypes { optional_nested_message: __ {} }); + assert_that!(msg.has_optional_nested_message(), eq(true)); } #[test] @@ -151,3 +167,38 @@ fn test_spread_nested_msg() { assert_that!(msg2.child().child().payload().optional_int32(), eq(42)); assert_that!(msg2.child().child().child().payload().optional_int32(), eq(43)); } + +#[test] +fn test_repeated_i32() { + let msg = proto!(TestAllTypes { repeated_int32: [1, 1 + 1, 3] }); + assert_that!(msg.repeated_int32().len(), eq(3)); + assert_that!(msg.repeated_int32().get(0).unwrap(), eq(1)); + assert_that!(msg.repeated_int32().get(1).unwrap(), eq(2)); + assert_that!(msg.repeated_int32().get(2).unwrap(), eq(3)); +} + +#[test] +fn test_repeated_msg() { + let msg2 = proto!(NestedTestAllTypes { payload: TestAllTypes { optional_int32: 1 } }); + let msg = proto!(NestedTestAllTypes { + child: NestedTestAllTypes { + repeated_child: [ + NestedTestAllTypes { payload: TestAllTypes { optional_int32: 0 } }, + msg2, + __ { payload: TestAllTypes { optional_int32: 2 } } + ] + }, + repeated_child: [ + __ { payload: __ { optional_int32: 1 } }, + NestedTestAllTypes { payload: TestAllTypes { optional_int32: 2 } } + ] + }); + assert_that!(msg.child().repeated_child().len(), eq(3)); + assert_that!(msg.child().repeated_child().get(0).unwrap().payload().optional_int32(), eq(0)); + assert_that!(msg.child().repeated_child().get(1).unwrap().payload().optional_int32(), eq(1)); + assert_that!(msg.child().repeated_child().get(2).unwrap().payload().optional_int32(), eq(2)); + + assert_that!(msg.repeated_child().len(), eq(2)); + assert_that!(msg.repeated_child().get(0).unwrap().payload().optional_int32(), eq(1)); + assert_that!(msg.repeated_child().get(1).unwrap().payload().optional_int32(), eq(2)); +} From 22863e1dfdf25ee0008b79b28b0991581b303df7 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 09:42:34 -0700 Subject: [PATCH 016/107] Move the `mutable` keyword into `CachedSize` to avoid spreading it all over. Makes the abstraction simpler to use. It has no effect in compiled code. PiperOrigin-RevId: 653655703 --- .../cpp/field_generators/enum_field.cc | 2 +- .../cpp/field_generators/primitive_field.cc | 2 +- src/google/protobuf/compiler/cpp/message.cc | 4 +- .../protobuf/compiler/java/java_features.pb.h | 2 +- src/google/protobuf/compiler/plugin.pb.h | 8 +-- src/google/protobuf/cpp_features.pb.h | 2 +- src/google/protobuf/descriptor.pb.h | 72 +++++++++---------- src/google/protobuf/dynamic_message.cc | 2 +- src/google/protobuf/generated_message_bases.h | 2 +- src/google/protobuf/implicit_weak_message.h | 2 +- src/google/protobuf/map_entry.h | 2 +- src/google/protobuf/message.cc | 4 +- src/google/protobuf/message.h | 8 +-- src/google/protobuf/message_lite.cc | 8 --- src/google/protobuf/message_lite.h | 14 ++-- 15 files changed, 65 insertions(+), 69 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc index e8d894c700..d8fd612f0f 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc @@ -241,7 +241,7 @@ class RepeatedEnum : public FieldGeneratorBase { if (has_cached_size_) { p->Emit(R"cc( - mutable $pbi$::CachedSize $cached_size_name$; + $pbi$::CachedSize $cached_size_name$; )cc"); } } diff --git a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc index 918d708554..76a2fb2d2d 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc @@ -437,7 +437,7 @@ void RepeatedPrimitive::GeneratePrivateMembers(io::Printer* p) const { if (HasCachedSize()) { p->Emit({{"_cached_size_", MakeVarintCachedSizeName(field_)}}, R"cc( - mutable $pbi$::CachedSize $_cached_size_$; + $pbi$::CachedSize $_cached_size_$; )cc"); } } diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 1f868e5897..4756acf4bc 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -1367,7 +1367,7 @@ void MessageGenerator::GenerateImplDefinition(io::Printer* p) { )cc"); if (need_to_emit_cached_size) { p->Emit(R"cc( - mutable ::$proto_ns$::internal::CachedSize _cached_size_; + ::$proto_ns$::internal::CachedSize _cached_size_; )cc"); need_to_emit_cached_size = false; } @@ -1437,7 +1437,7 @@ void MessageGenerator::GenerateImplDefinition(io::Printer* p) { need_to_emit_cached_size = false; p->Emit(R"cc( - mutable ::$proto_ns$::internal::CachedSize _cached_size_; + ::$proto_ns$::internal::CachedSize _cached_size_; )cc"); }}, {"oneof_case", diff --git a/src/google/protobuf/compiler/java/java_features.pb.h b/src/google/protobuf/compiler/java/java_features.pb.h index 5604c67cc5..2f851642ca 100644 --- a/src/google/protobuf/compiler/java/java_features.pb.h +++ b/src/google/protobuf/compiler/java/java_features.pb.h @@ -309,7 +309,7 @@ class PROTOC_EXPORT JavaFeatures final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const JavaFeatures& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; bool legacy_closed_enum_; int utf8_validation_; PROTOBUF_TSAN_DECLARE_MEMBER diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index ded79ec209..107bbbf8bd 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -336,7 +336,7 @@ class PROTOC_EXPORT Version final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Version& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr suffix_; ::int32_t major_; ::int32_t minor_; @@ -585,7 +585,7 @@ class PROTOC_EXPORT CodeGeneratorResponse_File final : public ::google::protobuf ::google::protobuf::Arena* arena, const Impl_& from, const CodeGeneratorResponse_File& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr insertion_point_; ::google::protobuf::internal::ArenaStringPtr content_; @@ -857,7 +857,7 @@ class PROTOC_EXPORT CodeGeneratorResponse final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const CodeGeneratorResponse& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::compiler::CodeGeneratorResponse_File > file_; ::google::protobuf::internal::ArenaStringPtr error_; ::uint64_t supported_features_; @@ -1135,7 +1135,7 @@ class PROTOC_EXPORT CodeGeneratorRequest final : public ::google::protobuf::Mess ::google::protobuf::Arena* arena, const Impl_& from, const CodeGeneratorRequest& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField file_to_generate_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FileDescriptorProto > proto_file_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FileDescriptorProto > source_file_descriptors_; diff --git a/src/google/protobuf/cpp_features.pb.h b/src/google/protobuf/cpp_features.pb.h index 8bfb1478cb..f5653ca1f6 100644 --- a/src/google/protobuf/cpp_features.pb.h +++ b/src/google/protobuf/cpp_features.pb.h @@ -311,7 +311,7 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const CppFeatures& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; bool legacy_closed_enum_; int string_type_; PROTOBUF_TSAN_DECLARE_MEMBER diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index a389d1d95a..6443a80622 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -903,7 +903,7 @@ class PROTOBUF_EXPORT UninterpretedOption_NamePart final : public ::google::prot ::google::protobuf::Arena* arena, const Impl_& from, const UninterpretedOption_NamePart& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_part_; bool is_extension_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -1177,11 +1177,11 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location final : public ::google::protobuf: ::google::protobuf::Arena* arena, const Impl_& from, const SourceCodeInfo_Location& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedField<::int32_t> path_; - mutable ::google::protobuf::internal::CachedSize _path_cached_byte_size_; + ::google::protobuf::internal::CachedSize _path_cached_byte_size_; ::google::protobuf::RepeatedField<::int32_t> span_; - mutable ::google::protobuf::internal::CachedSize _span_cached_byte_size_; + ::google::protobuf::internal::CachedSize _span_cached_byte_size_; ::google::protobuf::RepeatedPtrField leading_detached_comments_; ::google::protobuf::internal::ArenaStringPtr leading_comments_; ::google::protobuf::internal::ArenaStringPtr trailing_comments_; @@ -1452,9 +1452,9 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation final : public ::google::prot ::google::protobuf::Arena* arena, const Impl_& from, const GeneratedCodeInfo_Annotation& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedField<::int32_t> path_; - mutable ::google::protobuf::internal::CachedSize _path_cached_byte_size_; + ::google::protobuf::internal::CachedSize _path_cached_byte_size_; ::google::protobuf::internal::ArenaStringPtr source_file_; ::int32_t begin_; ::int32_t end_; @@ -1687,7 +1687,7 @@ class PROTOBUF_EXPORT FieldOptions_FeatureSupport final : public ::google::proto ::google::protobuf::Arena* arena, const Impl_& from, const FieldOptions_FeatureSupport& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr deprecation_warning_; int edition_introduced_; int edition_deprecated_; @@ -1896,7 +1896,7 @@ class PROTOBUF_EXPORT FieldOptions_EditionDefault final : public ::google::proto ::google::protobuf::Arena* arena, const Impl_& from, const FieldOptions_EditionDefault& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr value_; int edition_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -2451,7 +2451,7 @@ class PROTOBUF_EXPORT FeatureSet final : public ::google::protobuf::Message const FeatureSet& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; int field_presence_; int enum_type_; int repeated_field_encoding_; @@ -2704,7 +2704,7 @@ class PROTOBUF_EXPORT ExtensionRangeOptions_Declaration final : public ::google: ::google::protobuf::Arena* arena, const Impl_& from, const ExtensionRangeOptions_Declaration& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr full_name_; ::google::protobuf::internal::ArenaStringPtr type_; ::int32_t number_; @@ -2908,7 +2908,7 @@ class PROTOBUF_EXPORT EnumDescriptorProto_EnumReservedRange final : public ::goo ::google::protobuf::Arena* arena, const Impl_& from, const EnumDescriptorProto_EnumReservedRange& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::int32_t start_; ::int32_t end_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -3109,7 +3109,7 @@ class PROTOBUF_EXPORT DescriptorProto_ReservedRange final : public ::google::pro ::google::protobuf::Arena* arena, const Impl_& from, const DescriptorProto_ReservedRange& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::int32_t start_; ::int32_t end_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -3400,7 +3400,7 @@ class PROTOBUF_EXPORT UninterpretedOption final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const UninterpretedOption& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption_NamePart > name_; ::google::protobuf::internal::ArenaStringPtr identifier_value_; ::google::protobuf::internal::ArenaStringPtr string_value_; @@ -3601,7 +3601,7 @@ class PROTOBUF_EXPORT SourceCodeInfo final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const SourceCodeInfo& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::SourceCodeInfo_Location > location_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -3795,7 +3795,7 @@ class PROTOBUF_EXPORT GeneratedCodeInfo final : public ::google::protobuf::Messa ::google::protobuf::Arena* arena, const Impl_& from, const GeneratedCodeInfo& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::GeneratedCodeInfo_Annotation > annotation_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -4019,7 +4019,7 @@ class PROTOBUF_EXPORT FeatureSetDefaults_FeatureSetEditionDefault final : public ::google::protobuf::Arena* arena, const Impl_& from, const FeatureSetDefaults_FeatureSetEditionDefault& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::FeatureSet* overridable_features_; ::google::protobuf::FeatureSet* fixed_features_; int edition_; @@ -4428,7 +4428,7 @@ class PROTOBUF_EXPORT ServiceOptions final : public ::google::protobuf::Message const ServiceOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool deprecated_; @@ -4825,7 +4825,7 @@ class PROTOBUF_EXPORT OneofOptions final : public ::google::protobuf::Message const OneofOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -5265,7 +5265,7 @@ class PROTOBUF_EXPORT MethodOptions final : public ::google::protobuf::Message const MethodOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool deprecated_; @@ -5723,7 +5723,7 @@ class PROTOBUF_EXPORT MessageOptions final : public ::google::protobuf::Message const MessageOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool message_set_wire_format_; @@ -6432,7 +6432,7 @@ class PROTOBUF_EXPORT FileOptions final : public ::google::protobuf::Message const FileOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::internal::ArenaStringPtr java_package_; ::google::protobuf::internal::ArenaStringPtr java_outer_classname_; @@ -7098,7 +7098,7 @@ class PROTOBUF_EXPORT FieldOptions final : public ::google::protobuf::Message const FieldOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedField targets_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FieldOptions_EditionDefault > edition_defaults_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; @@ -7335,7 +7335,7 @@ class PROTOBUF_EXPORT FeatureSetDefaults final : public ::google::protobuf::Mess ::google::protobuf::Arena* arena, const Impl_& from, const FeatureSetDefaults& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FeatureSetDefaults_FeatureSetEditionDefault > defaults_; int minimum_edition_; int maximum_edition_; @@ -7782,7 +7782,7 @@ class PROTOBUF_EXPORT ExtensionRangeOptions final : public ::google::protobuf::M const ExtensionRangeOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::ExtensionRangeOptions_Declaration > declaration_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; @@ -8220,7 +8220,7 @@ class PROTOBUF_EXPORT EnumValueOptions final : public ::google::protobuf::Messag const EnumValueOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; ::google::protobuf::FieldOptions_FeatureSupport* feature_support_; @@ -8655,7 +8655,7 @@ class PROTOBUF_EXPORT EnumOptions final : public ::google::protobuf::Message const EnumOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool allow_alias_; @@ -8874,7 +8874,7 @@ class PROTOBUF_EXPORT OneofDescriptorProto final : public ::google::protobuf::Me ::google::protobuf::Arena* arena, const Impl_& from, const OneofDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::OneofOptions* options_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -9150,7 +9150,7 @@ class PROTOBUF_EXPORT MethodDescriptorProto final : public ::google::protobuf::M ::google::protobuf::Arena* arena, const Impl_& from, const MethodDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr input_type_; ::google::protobuf::internal::ArenaStringPtr output_type_; @@ -9557,7 +9557,7 @@ class PROTOBUF_EXPORT FieldDescriptorProto final : public ::google::protobuf::Me ::google::protobuf::Arena* arena, const Impl_& from, const FieldDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr extendee_; ::google::protobuf::internal::ArenaStringPtr type_name_; @@ -9794,7 +9794,7 @@ class PROTOBUF_EXPORT EnumValueDescriptorProto final : public ::google::protobuf ::google::protobuf::Arena* arena, const Impl_& from, const EnumValueDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::EnumValueOptions* options_; ::int32_t number_; @@ -10017,7 +10017,7 @@ class PROTOBUF_EXPORT DescriptorProto_ExtensionRange final : public ::google::pr ::google::protobuf::Arena* arena, const Impl_& from, const DescriptorProto_ExtensionRange& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::ExtensionRangeOptions* options_; ::int32_t start_; ::int32_t end_; @@ -10252,7 +10252,7 @@ class PROTOBUF_EXPORT ServiceDescriptorProto final : public ::google::protobuf:: ::google::protobuf::Arena* arena, const Impl_& from, const ServiceDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::MethodDescriptorProto > method_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::ServiceOptions* options_; @@ -10529,7 +10529,7 @@ class PROTOBUF_EXPORT EnumDescriptorProto final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const EnumDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumValueDescriptorProto > value_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumDescriptorProto_EnumReservedRange > reserved_range_; ::google::protobuf::RepeatedPtrField reserved_name_; @@ -10899,7 +10899,7 @@ class PROTOBUF_EXPORT DescriptorProto final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const DescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FieldDescriptorProto > field_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::DescriptorProto > nested_type_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumDescriptorProto > enum_type_; @@ -11320,7 +11320,7 @@ class PROTOBUF_EXPORT FileDescriptorProto final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const FileDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField dependency_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::DescriptorProto > message_type_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumDescriptorProto > enum_type_; @@ -11531,7 +11531,7 @@ class PROTOBUF_EXPORT FileDescriptorSet final : public ::google::protobuf::Messa ::google::protobuf::Arena* arena, const Impl_& from, const FileDescriptorSet& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::FileDescriptorProto > file_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 00b55f6031..138c941282 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -246,7 +246,7 @@ class DynamicMessage final : public Message { void* MutableOneofFieldRaw(const FieldDescriptor* f); const DynamicMessageFactory::TypeInfo* type_info_; - mutable internal::CachedSize cached_byte_size_; + internal::CachedSize cached_byte_size_; }; struct DynamicMessageFactory::TypeInfo { diff --git a/src/google/protobuf/generated_message_bases.h b/src/google/protobuf/generated_message_bases.h index 3dc5991d0c..48d94db6c4 100644 --- a/src/google/protobuf/generated_message_bases.h +++ b/src/google/protobuf/generated_message_bases.h @@ -51,7 +51,7 @@ class PROTOBUF_EXPORT ZeroFieldsBase : public Message { io::EpsCopyOutputStream* stream); struct { - mutable internal::CachedSize _cached_size_; + internal::CachedSize _cached_size_; } _impl_; }; diff --git a/src/google/protobuf/implicit_weak_message.h b/src/google/protobuf/implicit_weak_message.h index 457d34db1f..095028a6be 100644 --- a/src/google/protobuf/implicit_weak_message.h +++ b/src/google/protobuf/implicit_weak_message.h @@ -110,7 +110,7 @@ class PROTOBUF_EXPORT ImplicitWeakMessage final : public MessageLite { // the default instance can be constant-initialized. In the const methods, we // have to handle the possibility of data_ being null. std::string* data_; - mutable google::protobuf::internal::CachedSize cached_size_{}; + google::protobuf::internal::CachedSize cached_size_{}; }; struct ImplicitWeakMessageDefaultType; diff --git a/src/google/protobuf/map_entry.h b/src/google/protobuf/map_entry.h index 71dda48b9f..8704b63286 100644 --- a/src/google/protobuf/map_entry.h +++ b/src/google/protobuf/map_entry.h @@ -116,7 +116,7 @@ class MapEntry : public Message { // sharing easier. struct { HasBits<1> _has_bits_{}; - mutable CachedSize _cached_size_{}; + CachedSize _cached_size_{}; KeyOnMemory key_{KeyTypeHandler::Constinit()}; ValueOnMemory value_{ValueTypeHandler::Constinit()}; diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index 27c82f7869..f60818f424 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -185,7 +185,7 @@ size_t Message::ByteSizeLong() const { #endif // !PROTOBUF_CUSTOM_VTABLE size_t Message::ComputeUnknownFieldsSize( - size_t total_size, internal::CachedSize* cached_size) const { + size_t total_size, const internal::CachedSize* cached_size) const { total_size += WireFormat::ComputeUnknownFieldsSize( _internal_metadata_.unknown_fields( UnknownFieldSet::default_instance)); @@ -194,7 +194,7 @@ size_t Message::ComputeUnknownFieldsSize( } size_t Message::MaybeComputeUnknownFieldsSize( - size_t total_size, internal::CachedSize* cached_size) const { + size_t total_size, const internal::CachedSize* cached_size) const { if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { return ComputeUnknownFieldsSize(total_size, cached_size); } diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 45eeea451c..85bb049a30 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -383,10 +383,10 @@ class PROTOBUF_EXPORT Message : public MessageLite { // For CODE_SIZE types static bool IsInitializedImpl(const MessageLite&); - size_t ComputeUnknownFieldsSize(size_t total_size, - internal::CachedSize* cached_size) const; - size_t MaybeComputeUnknownFieldsSize(size_t total_size, - internal::CachedSize* cached_size) const; + size_t ComputeUnknownFieldsSize( + size_t total_size, const internal::CachedSize* cached_size) const; + size_t MaybeComputeUnknownFieldsSize( + size_t total_size, const internal::CachedSize* cached_size) const; // Reflection based version for reflection based types. static absl::string_view GetTypeNameImpl(const ClassData* data); diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 22fe7d9634..87ed3973f9 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -139,14 +139,6 @@ std::string MessageLite::DebugString() const { int MessageLite::GetCachedSize() const { return AccessCachedSize().Get(); } -internal::CachedSize& MessageLite::AccessCachedSize() const { - auto* data = GetClassData(); - ABSL_DCHECK(data != nullptr); - ABSL_DCHECK(data->cached_size_offset != 0); - return *reinterpret_cast(const_cast( - reinterpret_cast(this) + data->cached_size_offset)); -} - namespace { // When serializing, we first compute the byte size, then serialize the message. diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index cae4e6bbef..ba04789a2b 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -95,7 +95,7 @@ class PROTOBUF_EXPORT CachedSize { return __atomic_load_n(&atom_, __ATOMIC_RELAXED); } - void Set(Scalar desired) noexcept { + void Set(Scalar desired) const noexcept { __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); } #else @@ -109,16 +109,16 @@ class PROTOBUF_EXPORT CachedSize { return atom_.load(std::memory_order_relaxed); } - void Set(Scalar desired) noexcept { + void Set(Scalar desired) const noexcept { atom_.store(desired, std::memory_order_relaxed); } #endif private: #if PROTOBUF_BUILTIN_ATOMIC - Scalar atom_; + mutable Scalar atom_; #else - std::atomic atom_; + mutable std::atomic atom_; #endif }; @@ -770,7 +770,11 @@ class PROTOBUF_EXPORT MessageLite { // Return the cached size object as described by // ClassData::cached_size_offset. - internal::CachedSize& AccessCachedSize() const; + const internal::CachedSize& AccessCachedSize() const { + return *reinterpret_cast( + reinterpret_cast(this) + + GetClassData()->cached_size_offset); + } public: enum ParseFlags { From 01e8cc0b30a0e7defdddec24309b49a72f8a1524 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 16:57:29 +0000 Subject: [PATCH 017/107] Auto-generate files after cl/653655703 --- src/google/protobuf/any.pb.h | 2 +- src/google/protobuf/api.pb.h | 6 +++--- src/google/protobuf/duration.pb.h | 2 +- src/google/protobuf/field_mask.pb.h | 2 +- src/google/protobuf/source_context.pb.h | 2 +- src/google/protobuf/struct.pb.h | 6 +++--- src/google/protobuf/timestamp.pb.h | 2 +- src/google/protobuf/type.pb.h | 10 +++++----- src/google/protobuf/wrappers.pb.h | 18 +++++++++--------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 00ef47fd6f..79bea6da18 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -315,7 +315,7 @@ class PROTOBUF_EXPORT Any final : public ::google::protobuf::Message const Any& from_msg); ::google::protobuf::internal::ArenaStringPtr type_url_; ::google::protobuf::internal::ArenaStringPtr value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::AnyMetadata _any_metadata_; PROTOBUF_TSAN_DECLARE_MEMBER }; diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 8c34fc7768..d05be84724 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -275,7 +275,7 @@ class PROTOBUF_EXPORT Mixin final : public ::google::protobuf::Message const Mixin& from_msg); ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr root_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -558,7 +558,7 @@ class PROTOBUF_EXPORT Method final : public ::google::protobuf::Message bool request_streaming_; bool response_streaming_; int syntax_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -848,7 +848,7 @@ class PROTOBUF_EXPORT Api final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Api& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method > methods_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin > mixins_; diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 92e931a371..b1414b3f64 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -255,7 +255,7 @@ class PROTOBUF_EXPORT Duration final : public ::google::protobuf::Message const Duration& from_msg); ::int64_t seconds_; ::int32_t nanos_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 3b7114551f..b6f7e493c4 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -255,7 +255,7 @@ class PROTOBUF_EXPORT FieldMask final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const FieldMask& from_msg); ::google::protobuf::RepeatedPtrField paths_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 4f082b546e..806d61478c 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -249,7 +249,7 @@ class PROTOBUF_EXPORT SourceContext final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const SourceContext& from_msg); ::google::protobuf::internal::ArenaStringPtr file_name_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index e77dafad1c..9fceeb3759 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -295,7 +295,7 @@ class PROTOBUF_EXPORT ListValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const ListValue& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::Value > values_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -489,7 +489,7 @@ class PROTOBUF_EXPORT Struct final : public ::google::protobuf::Message ::google::protobuf::internal::WireFormatLite::TYPE_STRING, ::google::protobuf::internal::WireFormatLite::TYPE_MESSAGE> fields_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -820,7 +820,7 @@ class PROTOBUF_EXPORT Value final : public ::google::protobuf::Message ::google::protobuf::Struct* struct_value_; ::google::protobuf::ListValue* list_value_; } kind_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::uint32_t _oneof_case_[1]; PROTOBUF_TSAN_DECLARE_MEMBER }; diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 161829f1c6..1b9cc67ae9 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -255,7 +255,7 @@ class PROTOBUF_EXPORT Timestamp final : public ::google::protobuf::Message const Timestamp& from_msg); ::int64_t seconds_; ::int32_t nanos_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index e555ee56fe..21ecb391f6 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -399,7 +399,7 @@ class PROTOBUF_EXPORT Option final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Option& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::Any* value_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -783,7 +783,7 @@ class PROTOBUF_EXPORT Field final : public ::google::protobuf::Message ::int32_t number_; ::int32_t oneof_index_; bool packed_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1006,7 +1006,7 @@ class PROTOBUF_EXPORT EnumValue final : public ::google::protobuf::Message ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::internal::ArenaStringPtr name_; ::int32_t number_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1301,7 +1301,7 @@ class PROTOBUF_EXPORT Type final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Type& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Field > fields_; ::google::protobuf::RepeatedPtrField oneofs_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; @@ -1580,7 +1580,7 @@ class PROTOBUF_EXPORT Enum final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Enum& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumValue > enumvalue_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::internal::ArenaStringPtr name_; diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index bd627bee1a..6e156ce6f7 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -267,7 +267,7 @@ class PROTOBUF_EXPORT UInt64Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const UInt64Value& from_msg); ::uint64_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -453,7 +453,7 @@ class PROTOBUF_EXPORT UInt32Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const UInt32Value& from_msg); ::uint32_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -645,7 +645,7 @@ class PROTOBUF_EXPORT StringValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const StringValue& from_msg); ::google::protobuf::internal::ArenaStringPtr value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -831,7 +831,7 @@ class PROTOBUF_EXPORT Int64Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Int64Value& from_msg); ::int64_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1017,7 +1017,7 @@ class PROTOBUF_EXPORT Int32Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Int32Value& from_msg); ::int32_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1203,7 +1203,7 @@ class PROTOBUF_EXPORT FloatValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const FloatValue& from_msg); float value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1389,7 +1389,7 @@ class PROTOBUF_EXPORT DoubleValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const DoubleValue& from_msg); double value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1581,7 +1581,7 @@ class PROTOBUF_EXPORT BytesValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const BytesValue& from_msg); ::google::protobuf::internal::ArenaStringPtr value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1767,7 +1767,7 @@ class PROTOBUF_EXPORT BoolValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const BoolValue& from_msg); bool value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; From cf948e4a817c4340ae58602b21340d1dc67569b9 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 10:01:55 -0700 Subject: [PATCH 018/107] Restructure the ViewProxy versus MutProxy trait setup. Rather than two traits (MutProxy subtrait of ViewProxy), instead have three traits (MutProxy subtrait of Proxy, and ViewProxy subtrait of Proxy). This makes things more consistent, including that (MutProxied subtraits Proxied) is now more parallel to (MutProxy subtraits Proxy). ViewProxy is largely a marker trait here but leaves a spot for methods that should be on ViewProxies but not MutProxies if those do show up later. PiperOrigin-RevId: 653661953 --- rust/map.rs | 9 +++++++-- rust/primitive.rs | 6 ++++-- rust/proxied.rs | 20 +++++++++++++------- rust/repeated.rs | 8 +++++--- rust/shared.rs | 4 +++- rust/string.rs | 10 +++++++--- rust/test/shared/accessors_repeated_test.rs | 2 +- rust/upb.rs | 2 +- src/google/protobuf/compiler/rust/enum.cc | 4 +++- src/google/protobuf/compiler/rust/message.cc | 20 +++++++++++--------- 10 files changed, 55 insertions(+), 30 deletions(-) diff --git a/rust/map.rs b/rust/map.rs index bbb5bfc677..a0e0c9e449 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -6,7 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd use crate::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, + IntoProxied, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, __internal::Private, __runtime::{InnerMap, InnerMapMut, RawMap, RawMapIter}, }; @@ -112,7 +112,7 @@ impl + ?Sized> MutProxied for Map = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg> +impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> Proxy<'msg> for MapView<'msg, K, V> { type Proxied = Map; @@ -130,6 +130,11 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg } impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg> + for MapView<'msg, K, V> +{ +} + +impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> Proxy<'msg> for MapMut<'msg, K, V> { type Proxied = Map; diff --git a/rust/primitive.rs b/rust/primitive.rs index b5f4b38108..ab50352f14 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd use crate::__internal::Private; -use crate::{IntoProxied, Proxied, View, ViewProxy}; +use crate::{IntoProxied, Proxied, Proxy, View, ViewProxy}; macro_rules! impl_singular_primitives { ($($t:ty),*) => { @@ -14,7 +14,7 @@ macro_rules! impl_singular_primitives { type View<'msg> = $t; } - impl<'msg> ViewProxy<'msg> for $t { + impl<'msg> Proxy<'msg> for $t { type Proxied = $t; fn as_view(&self) -> View<'_, Self::Proxied> { @@ -26,6 +26,8 @@ macro_rules! impl_singular_primitives { } } + impl<'msg> ViewProxy<'msg> for $t {} + impl IntoProxied<$t> for $t { fn into_proxied(self, _private: Private) -> $t { self diff --git a/rust/proxied.rs b/rust/proxied.rs index b1903a4e43..e90dbe5805 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -56,7 +56,7 @@ pub trait Proxied: Sized { /// The proxy type that provides shared access to a `T`, like a `&'msg T`. /// /// Most code should use the type alias [`View`]. - type View<'msg>: ViewProxy<'msg, Proxied = Self> + Copy + Send + type View<'msg>: ViewProxy<'msg, Proxied = Self> where Self: 'msg; } @@ -90,11 +90,12 @@ pub type View<'msg, T> = ::View<'msg>; #[allow(dead_code)] pub type Mut<'msg, T> = ::Mut<'msg>; -/// Declares conversion operations common to all views. +/// Declares conversion operations common to all proxies (both views and mut +/// proxies). /// /// This trait is intentionally made non-object-safe to prevent a potential /// future incompatible change. -pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { +pub trait Proxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { type Proxied: 'msg + Proxied + ?Sized; /// Converts a borrow into a `View` with the lifetime of that borrow. @@ -148,11 +149,14 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { 'msg: 'shorter; } -/// Declares operations common to all mutators. +/// Declares conversion operations common to view proxies. +pub trait ViewProxy<'msg>: Proxy<'msg> + Copy + Send {} + +/// Declares operations common to all mut proxies. /// /// This trait is intentionally made non-object-safe to prevent a potential /// future incompatible change. -pub trait MutProxy<'msg>: ViewProxy<'msg> +pub trait MutProxy<'msg>: Proxy<'msg> where Self::Proxied: MutProxied, { @@ -259,7 +263,7 @@ mod tests { } } - impl<'msg> ViewProxy<'msg> for MyProxiedView<'msg> { + impl<'msg> Proxy<'msg> for MyProxiedView<'msg> { type Proxied = MyProxied; fn as_view(&self) -> View<'msg, MyProxied> { @@ -274,12 +278,14 @@ mod tests { } } + impl<'msg> ViewProxy<'msg> for MyProxiedView<'msg> {} + #[derive(Debug)] struct MyProxiedMut<'msg> { my_proxied_ref: &'msg mut MyProxied, } - impl<'msg> ViewProxy<'msg> for MyProxiedMut<'msg> { + impl<'msg> Proxy<'msg> for MyProxiedMut<'msg> { type Proxied = MyProxied; fn as_view(&self) -> View<'_, MyProxied> { diff --git a/rust/repeated.rs b/rust/repeated.rs index 99c935b4f4..692a24971f 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -15,7 +15,7 @@ use std::iter::FusedIterator; use std::marker::PhantomData; use crate::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, + IntoProxied, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, __internal::Private, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -397,7 +397,7 @@ where type Mut<'msg> = RepeatedMut<'msg, T> where Repeated: 'msg; } -impl<'msg, T> ViewProxy<'msg> for RepeatedView<'msg, T> +impl<'msg, T> Proxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg, { @@ -417,7 +417,9 @@ where } } -impl<'msg, T> ViewProxy<'msg> for RepeatedMut<'msg, T> +impl<'msg, T> ViewProxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> Proxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg, { diff --git a/rust/shared.rs b/rust/shared.rs index 4494ee7019..ec31d65a01 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -27,7 +27,9 @@ pub mod __public { pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; pub use crate::optional::Optional; pub use crate::proto; - pub use crate::proxied::{IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy}; + pub use crate::proxied::{ + IntoProxied, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, + }; pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, }; diff --git a/rust/string.rs b/rust/string.rs index a94f4efc00..6ebdd37af1 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -11,7 +11,7 @@ use crate::__internal::Private; use crate::__runtime::{InnerProtoString, PtrAndLen, RawMessage}; -use crate::{IntoProxied, Mut, MutProxied, MutProxy, Optional, Proxied, View, ViewProxy}; +use crate::{IntoProxied, Mut, MutProxied, MutProxy, Optional, Proxied, Proxy, View, ViewProxy}; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use std::convert::{AsMut, AsRef}; @@ -114,7 +114,7 @@ impl IntoProxied for Arc<[u8]> { } } -impl<'msg> ViewProxy<'msg> for &'msg [u8] { +impl<'msg> Proxy<'msg> for &'msg [u8] { type Proxied = ProtoBytes; fn as_view(&self) -> &[u8] { @@ -129,6 +129,8 @@ impl<'msg> ViewProxy<'msg> for &'msg [u8] { } } +impl<'msg> ViewProxy<'msg> for &'msg [u8] {} + /// The bytes were not valid UTF-8. #[derive(Debug, PartialEq)] pub struct Utf8Error(pub(crate) ()); @@ -479,7 +481,7 @@ impl Proxied for ProtoString { type View<'msg> = &'msg ProtoStr; } -impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { +impl<'msg> Proxy<'msg> for &'msg ProtoStr { type Proxied = ProtoString; fn as_view(&self) -> &ProtoStr { @@ -494,6 +496,8 @@ impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { } } +impl<'msg> ViewProxy<'msg> for &'msg ProtoStr {} + /// Implements `PartialCmp` and `PartialEq` for the `lhs` against the `rhs` /// using `AsRef<[u8]>`. // TODO: consider improving to not require a `<()>` if no generics are diff --git a/rust/test/shared/accessors_repeated_test.rs b/rust/test/shared/accessors_repeated_test.rs index 9f70074bf4..bfec487ce9 100644 --- a/rust/test/shared/accessors_repeated_test.rs +++ b/rust/test/shared/accessors_repeated_test.rs @@ -7,7 +7,7 @@ use googletest::prelude::*; use paste::paste; -use protobuf::ViewProxy; +use protobuf::Proxy; use unittest_rust_proto::{test_all_types, test_all_types::NestedMessage, TestAllTypes}; macro_rules! generate_repeated_numeric_test { diff --git a/rust/upb.rs b/rust/upb.rs index a9c5581214..2c0084da36 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -10,7 +10,7 @@ use crate::__internal::{Enum, Private}; use crate::{ IntoProxied, Map, MapIter, MapMut, MapView, Mut, ProtoBytes, ProtoStr, ProtoString, Proxied, - ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, ViewProxy, + ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, }; use core::fmt::Debug; use std::alloc::Layout; diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 74613aa8f2..a2b1143146 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -405,7 +405,7 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { type View<'a> = $name$; } - impl $pb$::ViewProxy<'_> for $name$ { + impl $pb$::Proxy<'_> for $name$ { type Proxied = $name$; fn as_view(&self) -> $name$ { @@ -417,6 +417,8 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } } + impl $pb$::ViewProxy<'_> for $name$ {} + unsafe impl $pb$::ProxiedInRepeated for $name$ { fn repeated_new(_private: $pbi$::Private) -> $pb$::Repeated { $pbr$::new_enum_repeated($pbi$::Private) diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 54cac79093..20217a1ae5 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -260,7 +260,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - $pb$::IntoProxied::into_proxied($pb$::ViewProxy::into_view(self), _private) + $pb$::IntoProxied::into_proxied($pb$::Proxy::into_view(self), _private) } } @@ -289,7 +289,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - $pb$::IntoProxied::into_proxied($pb$::ViewProxy::into_view(self), _private) + $pb$::IntoProxied::into_proxied($pb$::Proxy::into_view(self), _private) } } @@ -323,7 +323,7 @@ void MessageMergeFrom(Context& ctx, const Descriptor& msg) { {"merge_from_thunk", ThunkName(ctx, msg, "merge_from")}, }, R"rs( - pub fn merge_from<'src>(&mut self, src: impl $pb$::ViewProxy<'src, Proxied = $Msg$>) { + pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { // SAFETY: self and src are both valid `$Msg$`s. unsafe { $merge_from_thunk$(self.raw_msg(), src.as_view().raw_msg()); @@ -337,7 +337,7 @@ void MessageMergeFrom(Context& ctx, const Descriptor& msg) { {"minitable", UpbMinitableName(msg)}, }, R"rs( - pub fn merge_from<'src>(&mut self, src: impl $pb$::ViewProxy<'src, Proxied = $Msg$>) { + pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { // SAFETY: self and src are both valid `$Msg$`s. unsafe { assert!( @@ -1029,7 +1029,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { // - `$Msg$View` does not use thread-local data. unsafe impl Send for $Msg$View<'_> {} - impl<'msg> $pb$::ViewProxy<'msg> for $Msg$View<'msg> { + impl<'msg> $pb$::Proxy<'msg> for $Msg$View<'msg> { type Proxied = $Msg$; fn as_view(&self) -> $pb$::View<'msg, $Msg$> { @@ -1040,6 +1040,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl<'msg> $pb$::ViewProxy<'msg> for $Msg$View<'msg> {} + $into_proxied_impl$ $repeated_impl$ @@ -1088,11 +1090,11 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } pub fn serialize(&self) -> Result, $pb$::SerializeError> { - $pb$::ViewProxy::as_view(self).serialize() + $pb$::Proxy::as_view(self).serialize() } pub fn to_owned(&self) -> $Msg$ { - $pb$::ViewProxy::as_view(self).to_owned() + $pb$::Proxy::as_view(self).to_owned() } $msg_merge_from$ @@ -1117,7 +1119,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { fn into_mut<'shorter>(self) -> $pb$::Mut<'shorter, $Msg$> where 'msg : 'shorter { self } } - impl<'msg> $pb$::ViewProxy<'msg> for $Msg$Mut<'msg> { + impl<'msg> $pb$::Proxy<'msg> for $Msg$Mut<'msg> { type Proxied = $Msg$; fn as_view(&self) -> $pb$::View<'_, $Msg$> { $Msg$View { msg: self.raw_msg(), _phantom: $std$::marker::PhantomData } @@ -1166,7 +1168,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $Msg$Mut::new($pbi$::Private, &mut self.inner) } - pub fn merge_from<'src>(&mut self, src: impl $pb$::ViewProxy<'src, Proxied = $Msg$>) { + pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { self.as_mut().merge_from(src); } From b4a7757369cbc285e04e220b26e4d8c0b8013c71 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 18 Jul 2024 10:28:50 -0700 Subject: [PATCH 019/107] Further refactoring in preparation for RepeatedField SOO: avoid repeated calls to accessors within the same functions. Motivation: these accessors (e.g. size(), Capacity()) will have branches on `is_soo()` so it's best to avoid calling them repeatedly if we can instead save the result in a local variable. Also update some comments to avoid referring to the names of data members that will no longer exist with SOO. PiperOrigin-RevId: 653672810 --- src/google/protobuf/repeated_field.h | 100 ++++++++++-------- .../protobuf/repeated_field_unittest.cc | 4 +- 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 151c0b6cfc..66c2351fae 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -387,9 +387,9 @@ class RepeatedField final // Reserves space to expand the field to at least the given size. // If the array is grown, it will always be at least doubled in size. // If `annotate_size` is true (the default), then this function will annotate - // the old container from `old_size` to `capacity_` (unpoison memory) + // the old container from `old_size` to `Capacity()` (unpoison memory) // directly before it is being released, and annotate the new container from - // `capacity_` to `old_size` (poison unused memory). + // `Capacity()` to `old_size` (poison unused memory). void Grow(int old_size, int new_size); void GrowNoAnnotate(int old_size, int new_size); @@ -410,10 +410,10 @@ class RepeatedField final } } - // Replaces size_ with new_size and returns the previous value of - // size_. This function is intended to be the only place where - // size_ is modified, with the exception of `AddInputIterator()` - // where the size of added items is not known in advance. + // Replaces size with new_size and returns the previous value of size. This + // function is intended to be the only place where size is modified, with the + // exception of `AddInputIterator()` where the size of added items is not + // known in advance. inline int ExchangeCurrentSize(int new_size) { const int prev_size = size(); AnnotateSize(prev_size, new_size); @@ -422,7 +422,7 @@ class RepeatedField final } // Returns a pointer to elements array. - // pre-condition: the array must have been allocated. + // pre-condition: Capacity() > 0. Element* elements() const { ABSL_DCHECK_GT(Capacity(), 0); // Because of above pre-condition this cast is safe. @@ -612,12 +612,14 @@ inline Element* RepeatedField::AddNAlreadyReserved(int n) template inline void RepeatedField::Resize(int new_size, const Element& value) { ABSL_DCHECK_GE(new_size, 0); - if (new_size > size()) { - if (new_size > Capacity()) Grow(size(), new_size); + const int old_size = size(); + if (new_size > old_size) { + if (new_size > Capacity()) Grow(old_size, new_size); Element* first = elements() + ExchangeCurrentSize(new_size); - std::uninitialized_fill(first, elements() + size(), value); - } else if (new_size < size()) { - Destroy(unsafe_elements() + new_size, unsafe_elements() + size()); + std::uninitialized_fill(first, elements() + new_size, value); + } else if (new_size < old_size) { + Element* elem = unsafe_elements(); + Destroy(elem + new_size, elem + old_size); ExchangeCurrentSize(new_size); } } @@ -663,14 +665,15 @@ inline void RepeatedField::Set(int index, const Element& value) { template inline void RepeatedField::Add(Element value) { + const int old_size = size(); int capacity = Capacity(); Element* elem = unsafe_elements(); - if (ABSL_PREDICT_FALSE(size() == capacity)) { - Grow(size(), size() + 1); + if (ABSL_PREDICT_FALSE(old_size == capacity)) { + Grow(old_size, old_size + 1); capacity = Capacity(); elem = unsafe_elements(); } - int new_size = size() + 1; + int new_size = old_size + 1; void* p = elem + ExchangeCurrentSize(new_size); ::new (p) Element(std::move(value)); @@ -682,21 +685,23 @@ inline void RepeatedField::Add(Element value) { template inline Element* RepeatedField::Add() ABSL_ATTRIBUTE_LIFETIME_BOUND { - if (ABSL_PREDICT_FALSE(size() == Capacity())) { - Grow(size(), size() + 1); + const int old_size = size(); + if (ABSL_PREDICT_FALSE(old_size == Capacity())) { + Grow(old_size, old_size + 1); } - void* p = unsafe_elements() + ExchangeCurrentSize(size() + 1); + void* p = unsafe_elements() + ExchangeCurrentSize(old_size + 1); return ::new (p) Element; } template template inline void RepeatedField::AddForwardIterator(Iter begin, Iter end) { + const int old_size = size(); int capacity = Capacity(); Element* elem = unsafe_elements(); - int new_size = size() + static_cast(std::distance(begin, end)); + int new_size = old_size + static_cast(std::distance(begin, end)); if (ABSL_PREDICT_FALSE(new_size > capacity)) { - Grow(size(), new_size); + Grow(old_size, new_size); elem = unsafe_elements(); capacity = Capacity(); } @@ -711,24 +716,27 @@ inline void RepeatedField::AddForwardIterator(Iter begin, Iter end) { template template inline void RepeatedField::AddInputIterator(Iter begin, Iter end) { - Element* first = unsafe_elements() + size(); - Element* last = unsafe_elements() + Capacity(); + Element* elem = unsafe_elements(); + Element* first = elem + size(); + Element* last = elem + Capacity(); AnnotateSize(size(), Capacity()); while (begin != end) { if (ABSL_PREDICT_FALSE(first == last)) { - int size = first - unsafe_elements(); + int size = first - elem; GrowNoAnnotate(size, size + 1); - first = unsafe_elements() + size; - last = unsafe_elements() + Capacity(); + elem = unsafe_elements(); + first = elem + size; + last = elem + Capacity(); } ::new (static_cast(first)) Element(*begin); ++begin; ++first; } - set_size(first - unsafe_elements()); - AnnotateSize(Capacity(), size()); + const int new_size = first - elem; + set_size(new_size); + AnnotateSize(Capacity(), new_size); } template @@ -745,9 +753,10 @@ inline void RepeatedField::Add(Iter begin, Iter end) { template inline void RepeatedField::RemoveLast() { - ABSL_DCHECK_GT(size(), 0); - elements()[size() - 1].~Element(); - ExchangeCurrentSize(size() - 1); + const int old_size = size(); + ABSL_DCHECK_GT(old_size, 0); + elements()[old_size - 1].~Element(); + ExchangeCurrentSize(old_size - 1); } template @@ -755,7 +764,8 @@ void RepeatedField::ExtractSubrange(int start, int num, Element* elements) { ABSL_DCHECK_GE(start, 0); ABSL_DCHECK_GE(num, 0); - ABSL_DCHECK_LE(start + num, size()); + const int old_size = size(); + ABSL_DCHECK_LE(start + num, old_size); // Save the values of the removed elements if requested. if (elements != nullptr) { @@ -764,24 +774,26 @@ void RepeatedField::ExtractSubrange(int start, int num, // Slide remaining elements down to fill the gap. if (num > 0) { - for (int i = start + num; i < size(); ++i) Set(i - num, Get(i)); - Truncate(size() - num); + for (int i = start + num; i < old_size; ++i) Set(i - num, Get(i)); + Truncate(old_size - num); } } template inline void RepeatedField::Clear() { - Destroy(unsafe_elements(), unsafe_elements() + size()); + Element* elem = unsafe_elements(); + Destroy(elem, elem + size()); ExchangeCurrentSize(0); } template inline void RepeatedField::MergeFrom(const RepeatedField& other) { ABSL_DCHECK_NE(&other, this); - if (auto size = other.size()) { - Reserve(this->size() + size); - Element* dst = elements() + ExchangeCurrentSize(this->size() + size); - UninitializedCopyN(other.elements(), size, dst); + if (auto other_size = other.size()) { + const int old_size = size(); + Reserve(old_size + other_size); + Element* dst = elements() + ExchangeCurrentSize(old_size + other_size); + UninitializedCopyN(other.elements(), other_size, dst); } } @@ -905,8 +917,8 @@ RepeatedField::cend() const ABSL_ATTRIBUTE_LIFETIME_BOUND { template inline size_t RepeatedField::SpaceUsedExcludingSelfLong() const { - return Capacity() > 0 ? (Capacity() * sizeof(Element) + kHeapRepHeaderSize) - : 0; + const int capacity = Capacity(); + return capacity > 0 ? capacity * sizeof(Element) + kHeapRepHeaderSize : 0; } namespace internal { @@ -1016,9 +1028,11 @@ PROTOBUF_NOINLINE void RepeatedField::Grow(int old_size, template inline void RepeatedField::Truncate(int new_size) { - ABSL_DCHECK_LE(new_size, size()); - if (new_size < size()) { - Destroy(unsafe_elements() + new_size, unsafe_elements() + size()); + const int old_size = size(); + ABSL_DCHECK_LE(new_size, old_size); + if (new_size < old_size) { + Element* elem = unsafe_elements(); + Destroy(elem + new_size, elem + old_size); ExchangeCurrentSize(new_size); } } diff --git a/src/google/protobuf/repeated_field_unittest.cc b/src/google/protobuf/repeated_field_unittest.cc index 4b4d328111..50e43f594e 100644 --- a/src/google/protobuf/repeated_field_unittest.cc +++ b/src/google/protobuf/repeated_field_unittest.cc @@ -1241,8 +1241,8 @@ TEST(RepeatedField, HardenAgainstBadTruncate) { for (int size = 0; size < 10; ++size) { field.Truncate(size); #if GTEST_HAS_DEATH_TEST - EXPECT_DEBUG_DEATH(field.Truncate(size + 1), "new_size <= size"); - EXPECT_DEBUG_DEATH(field.Truncate(size + 2), "new_size <= size"); + EXPECT_DEBUG_DEATH(field.Truncate(size + 1), "new_size <= old_size"); + EXPECT_DEBUG_DEATH(field.Truncate(size + 2), "new_size <= old_size"); #elif defined(NDEBUG) field.Truncate(size + 1); field.Truncate(size + 1); From d0e49dfe3161714046a4ee2aeafba133b171e26f Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 18 Jul 2024 10:41:30 -0700 Subject: [PATCH 020/107] Introduce FieldDescriptor::cpp_string_type() API to replace direct ctype inspection which will be removed in the next breaking change This should provide the roughly same result as ctype, except that it reflects actual behavior rather than the specification in the proto file. VIEW will now be visible, and some subtleties around CORD and PIECE will change. PiperOrigin-RevId: 653677655 --- src/google/protobuf/descriptor.cc | 23 +++++++ src/google/protobuf/descriptor.h | 19 +++--- src/google/protobuf/descriptor_lite.h | 11 ++++ src/google/protobuf/descriptor_unittest.cc | 70 ++++++++++++++++++++++ 4 files changed, 115 insertions(+), 8 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index ed5f6af920..4561431797 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3939,6 +3939,29 @@ bool FieldDescriptor::has_optional_keyword() const { is_optional() && !containing_oneof()); } +FieldDescriptor::CppStringType FieldDescriptor::cpp_string_type() const { + ABSL_DCHECK(cpp_type() == FieldDescriptor::CPPTYPE_STRING); + switch (features().GetExtension(pb::cpp).string_type()) { + case pb::CppFeatures::VIEW: + return CppStringType::kView; + case pb::CppFeatures::CORD: + // In open-source, protobuf CORD is only supported for singular bytes + // fields. + if (type() != FieldDescriptor::TYPE_BYTES || is_repeated() || + is_extension()) { + return CppStringType::kString; + } + return CppStringType::kCord; + case pb::CppFeatures::STRING: + return CppStringType::kString; + default: + // If features haven't been resolved, this is a dynamic build not for C++ + // codegen. Just use string type. + ABSL_DCHECK(!features().GetExtension(pb::cpp).has_string_type()); + return CppStringType::kString; + } +} + // Location methods =============================================== bool FileDescriptor::GetSourceLocation(const std::vector& path, diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 28db34d95e..08a3d5c510 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -875,6 +875,10 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase, const char* cpp_type_name() const; // Name of the C++ type. Label label() const; // optional/required/repeated +#ifndef SWIG + CppStringType cpp_string_type() const; // The C++ string type of this field. +#endif + bool is_required() const; // shorthand for label() == LABEL_REQUIRED bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL bool is_repeated() const; // shorthand for label() == LABEL_REPEATED @@ -2932,22 +2936,21 @@ PROTOBUF_EXPORT bool HasPreservingUnknownEnumSemantics( PROTOBUF_EXPORT bool HasHasbit(const FieldDescriptor* field); +#ifndef SWIG // For a string field, returns the effective ctype. If the actual ctype is // not supported, returns the default of STRING. template typename FieldOpts::CType EffectiveStringCType(const FieldDesc* field) { - ABSL_DCHECK(field->cpp_type() == FieldDescriptor::CPPTYPE_STRING); - // Open-source protobuf release only supports STRING ctype and CORD for - // sinuglar bytes. - if (field->type() == FieldDescriptor::TYPE_BYTES && !field->is_repeated() && - field->options().ctype() == FieldOpts::CORD && !field->is_extension()) { - return FieldOpts::CORD; + // TODO Replace this function with FieldDescriptor::string_type; + switch (field->cpp_string_type()) { + case FieldDescriptor::CppStringType::kCord: + return FieldOpts::CORD; + default: + return FieldOpts::STRING; } - return FieldOpts::STRING; } -#ifndef SWIG enum class Utf8CheckMode : uint8_t { kStrict = 0, // Parsing will fail if non UTF-8 data is in string fields. kVerify = 1, // Only log an error but parsing will succeed. diff --git a/src/google/protobuf/descriptor_lite.h b/src/google/protobuf/descriptor_lite.h index db5805affe..e1a90b7fa6 100644 --- a/src/google/protobuf/descriptor_lite.h +++ b/src/google/protobuf/descriptor_lite.h @@ -78,6 +78,17 @@ class FieldDescriptorLite { MAX_LABEL = 3, // Constant useful for defining lookup tables // indexed by Label. }; + + // Identifies the storage type of a C++ string field. This corresponds to + // pb.CppFeatures.StringType, but is compatible with ctype prior to Edition + // 2024. 0 is reserved for errors. +#ifndef SWIG + enum class CppStringType { + kView = 1, + kCord = 2, + kString = 3, + }; +#endif }; } // namespace internal diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 0c13b3a2b4..f4d3137099 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -7594,6 +7594,9 @@ TEST_F(FeaturesTest, Proto2Features) { EXPECT_TRUE(message->FindFieldByName("req")->is_required()); EXPECT_TRUE(file->enum_type(0)->is_closed()); + EXPECT_EQ(message->FindFieldByName("str")->cpp_string_type(), + FieldDescriptor::CppStringType::kString); + // Check round-trip consistency. FileDescriptorProto proto; file->CopyTo(&proto); @@ -9710,6 +9713,73 @@ TEST_F(FeaturesTest, EnumFeatureHelpers) { EXPECT_FALSE(HasPreservingUnknownEnumSemantics(field_legacy_closed)); } +TEST_F(FeaturesTest, FieldCppStringType) { + BuildDescriptorMessagesInTestPool(); + const std::string file_contents = absl::Substitute( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2024 + message_type { + name: "Foo" + field { + name: "view" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + } + field { + name: "str" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_STRING + options { + features { + [pb.cpp] { string_type: STRING } + } + } + } + field { + name: "cord" + number: 3 + label: LABEL_OPTIONAL + type: TYPE_STRING + options { + features { + [pb.cpp] { string_type: CORD } + } + } + } + field { + name: "cord_bytes" + number: 4 + label: LABEL_OPTIONAL + type: TYPE_BYTES + options { + features { + [pb.cpp] { string_type: CORD } + } + } + } $0 + } + )pb", + "" + ); + const FileDescriptor* file = BuildFile(file_contents); + const Descriptor* message = file->message_type(0); + const FieldDescriptor* view = message->field(0); + const FieldDescriptor* str = message->field(1); + const FieldDescriptor* cord = message->field(2); + const FieldDescriptor* cord_bytes = message->field(3); + + EXPECT_EQ(view->cpp_string_type(), FieldDescriptor::CppStringType::kView); + EXPECT_EQ(str->cpp_string_type(), FieldDescriptor::CppStringType::kString); + EXPECT_EQ(cord_bytes->cpp_string_type(), + FieldDescriptor::CppStringType::kCord); + EXPECT_EQ(cord->cpp_string_type(), FieldDescriptor::CppStringType::kString); + +} + TEST_F(FeaturesTest, MergeFeatureValidationFailed) { BuildDescriptorMessagesInTestPool(); BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); From b84942022d6efefe22f46c703bf61a542e0b6fcd Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 10:48:34 -0700 Subject: [PATCH 021/107] Internal change. PiperOrigin-RevId: 653680509 --- src/google/protobuf/unknown_field_set.cc | 18 ++++++------- src/google/protobuf/unknown_field_set.h | 25 ++++++------------- .../protobuf/unknown_field_set_unittest.cc | 4 --- 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index 852a73f2d6..74fbb8f1b2 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -26,7 +26,6 @@ #include "google/protobuf/wire_format.h" #include "google/protobuf/wire_format_lite.h" - // Must be included last. #include "google/protobuf/port_def.inc" @@ -95,9 +94,9 @@ size_t UnknownFieldSet::SpaceUsedExcludingSelfLong() const { for (const UnknownField& field : fields_) { switch (field.type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - total_size += sizeof(*field.data_.length_delimited_.string_value) + + total_size += sizeof(*field.data_.string_value) + internal::StringSpaceUsedExcludingSelfLong( - *field.data_.length_delimited_.string_value); + *field.data_.string_value); break; case UnknownField::TYPE_GROUP: total_size += field.data_.group_->SpaceUsedLong(); @@ -142,11 +141,10 @@ std::string* UnknownFieldSet::AddLengthDelimited(int number) { auto& field = fields_.back(); field.number_ = number; field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); - field.data_.length_delimited_.string_value = new std::string; - return field.data_.length_delimited_.string_value; + field.data_.string_value = new std::string; + return field.data_.string_value; } - UnknownFieldSet* UnknownFieldSet::AddGroup(int number) { fields_.emplace_back(); auto& field = fields_.back(); @@ -249,7 +247,7 @@ bool UnknownFieldSet::SerializeToCord(absl::Cord* output) const { void UnknownField::Delete() { switch (type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - delete data_.length_delimited_.string_value; + delete data_.string_value; break; case UnknownField::TYPE_GROUP: delete data_.group_; @@ -263,8 +261,7 @@ void UnknownField::DeepCopy(const UnknownField& other) { (void)other; // Parameter is used by Google-internal code. switch (type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - data_.length_delimited_.string_value = - new std::string(*data_.length_delimited_.string_value); + data_.string_value = new std::string(*data_.string_value); break; case UnknownField::TYPE_GROUP: { UnknownFieldSet* group = new UnknownFieldSet(); @@ -277,11 +274,10 @@ void UnknownField::DeepCopy(const UnknownField& other) { } } - uint8_t* UnknownField::InternalSerializeLengthDelimitedNoTag( uint8_t* target, io::EpsCopyOutputStream* stream) const { ABSL_DCHECK_EQ(TYPE_LENGTH_DELIMITED, type()); - const absl::string_view data = *data_.length_delimited_.string_value; + const absl::string_view data = *data_.string_value; target = io::CodedOutputStream::WriteVarint32ToArray(data.size(), target); target = stream->WriteRaw(data.data(), data.size(), target); return target; diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index e985b0f4d4..a58c36478d 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -255,6 +255,8 @@ class PROTOBUF_EXPORT UnknownField { uint8_t* InternalSerializeLengthDelimitedNoTag( uint8_t* target, io::EpsCopyOutputStream* stream) const; + private: + friend class UnknownFieldSet; // If this UnknownField contains a pointer, delete it. void Delete(); @@ -266,17 +268,13 @@ class PROTOBUF_EXPORT UnknownField { // UnknownField is being created. inline void SetType(Type type); - union LengthDelimited { - std::string* string_value; - }; - uint32_t number_; uint32_t type_; union { uint64_t varint_; uint32_t fixed32_; uint64_t fixed64_; - mutable union LengthDelimited length_delimited_; + std::string* string_value; UnknownFieldSet* group_; } data_; }; @@ -317,9 +315,6 @@ inline void UnknownFieldSet::AddLengthDelimited(int number, AddLengthDelimited(number)->assign(value.data(), value.size()); } - - - inline int UnknownField::number() const { return static_cast(number_); } inline UnknownField::Type UnknownField::type() const { return static_cast(type_); @@ -339,7 +334,7 @@ inline uint64_t UnknownField::fixed64() const { } inline internal::UFSStringView UnknownField::length_delimited() const { assert(type() == TYPE_LENGTH_DELIMITED); - return *data_.length_delimited_.string_value; + return *data_.string_value; } inline const UnknownFieldSet& UnknownField::group() const { assert(type() == TYPE_GROUP); @@ -360,11 +355,11 @@ inline void UnknownField::set_fixed64(uint64_t value) { } inline void UnknownField::set_length_delimited(const absl::string_view value) { assert(type() == TYPE_LENGTH_DELIMITED); - data_.length_delimited_.string_value->assign(value.data(), value.size()); + data_.string_value->assign(value.data(), value.size()); } inline std::string* UnknownField::mutable_length_delimited() { assert(type() == TYPE_LENGTH_DELIMITED); - return data_.length_delimited_.string_value; + return data_.string_value; } inline UnknownFieldSet* UnknownField::mutable_group() { assert(type() == TYPE_GROUP); @@ -376,16 +371,12 @@ bool UnknownFieldSet::MergeFromMessage(const MessageType& message) { return InternalMergeFromMessage(message); } - inline size_t UnknownField::GetLengthDelimitedSize() const { ABSL_DCHECK_EQ(TYPE_LENGTH_DELIMITED, type()); - return data_.length_delimited_.string_value->size(); -} - -inline void UnknownField::SetType(Type type) { - type_ = type; + return data_.string_value->size(); } +inline void UnknownField::SetType(Type type) { type_ = type; } } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/unknown_field_set_unittest.cc b/src/google/protobuf/unknown_field_set_unittest.cc index dcd20e5ad6..4e926123d6 100644 --- a/src/google/protobuf/unknown_field_set_unittest.cc +++ b/src/google/protobuf/unknown_field_set_unittest.cc @@ -37,7 +37,6 @@ #include "google/protobuf/unittest_lite.pb.h" #include "google/protobuf/wire_format.h" - namespace google { namespace protobuf { @@ -335,7 +334,6 @@ TEST_F(UnknownFieldSetTest, MergeFromMessageLite) { EXPECT_EQ(unknown_field.fixed32(), 42); } - TEST_F(UnknownFieldSetTest, Clear) { // Clear the set. empty_message_.Clear(); @@ -575,7 +573,6 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var2"; } - TEST_F(UnknownFieldSetTest, Empty) { UnknownFieldSet unknown_fields; EXPECT_TRUE(unknown_fields.empty()); @@ -732,6 +729,5 @@ TEST_F(UnknownFieldSetTest, SerializeToCord_TestPackedTypes) { } } // namespace - } // namespace protobuf } // namespace google From bf8a9fc73f268a793fb43c2d6d2cf7595c40c4ec Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 11:09:25 -0700 Subject: [PATCH 022/107] Make inline strings constant initialized when supported by language version. PiperOrigin-RevId: 653688195 --- src/google/protobuf/inlined_string_field.h | 33 +++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/google/protobuf/inlined_string_field.h b/src/google/protobuf/inlined_string_field.h index d894acb7ec..25c7a7a573 100644 --- a/src/google/protobuf/inlined_string_field.h +++ b/src/google/protobuf/inlined_string_field.h @@ -85,16 +85,28 @@ namespace internal { // For more details of the donating states transitions, go/pd-inlined-string. class PROTOBUF_EXPORT InlinedStringField { public: - InlinedStringField() { Init(); } + InlinedStringField() : str_() {} InlinedStringField(const InlinedStringField&) = delete; InlinedStringField& operator=(const InlinedStringField&) = delete; - inline void Init() { new (get_mutable()) std::string(); } +#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L + // No need to do dynamic initialization here. + constexpr void Init() {} // Add the dummy parameter just to make InlinedStringField(nullptr) // unambiguous. constexpr InlinedStringField( const ExplicitlyConstructed* /*default_value*/, bool /*dummy*/) - : value_{} {} + : str_{} {} +#else + inline void Init() { ::new (static_cast(&str_)) std::string(); } + // Add the dummy parameter just to make InlinedStringField(nullptr) + // unambiguous. + constexpr InlinedStringField( + const ExplicitlyConstructed* /*default_value*/, + bool /*dummy*/) + : dummy_{} {} +#endif + explicit InlinedStringField(const std::string& default_value); explicit InlinedStringField(Arena* arena); InlinedStringField(Arena* arena, const InlinedStringField& rhs); @@ -346,7 +358,10 @@ class PROTOBUF_EXPORT InlinedStringField { PROTOBUF_NDEBUG_INLINE std::string* get_mutable(); PROTOBUF_NDEBUG_INLINE const std::string* get_const() const; - alignas(std::string) char value_[sizeof(std::string)]; + union { + std::string str_; + char dummy_; + }; std::string* MutableSlow(::google::protobuf::Arena* arena, bool donated, uint32_t* donating_states, uint32_t mask, @@ -359,12 +374,10 @@ class PROTOBUF_EXPORT InlinedStringField { typedef void DestructorSkippable_; }; -inline std::string* InlinedStringField::get_mutable() { - return reinterpret_cast(&value_); -} +inline std::string* InlinedStringField::get_mutable() { return &str_; } inline const std::string* InlinedStringField::get_const() const { - return reinterpret_cast(&value_); + return &str_; } inline InlinedStringField::InlinedStringField( @@ -386,12 +399,12 @@ inline void InternalRegisterArenaDtor(Arena* arena, void* object, } #endif // GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL_INLINE -inline InlinedStringField::InlinedStringField(Arena* /*arena*/) { Init(); } +inline InlinedStringField::InlinedStringField(Arena* /*arena*/) : str_() {} inline InlinedStringField::InlinedStringField(Arena* arena, const InlinedStringField& rhs) { const std::string& src = *rhs.get_const(); - new (value_) std::string(src); + ::new (static_cast(&str_)) std::string(src); } inline const std::string& InlinedStringField::GetNoArena() const { From 4ea1f2024a3b37de97efff896f9c030309170167 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 11:42:03 -0700 Subject: [PATCH 023/107] Move ByteSizeLong calculation of packed varint fields out of line. This reduces code size. PiperOrigin-RevId: 653700178 --- .../cpp/field_generators/enum_field.cc | 45 +++++----- .../cpp/field_generators/primitive_field.cc | 81 +++++++++-------- src/google/protobuf/descriptor.pb.cc | 67 ++++----------- src/google/protobuf/wire_format_lite.cc | 86 +++++++++++++++++++ src/google/protobuf/wire_format_lite.h | 22 +++++ 5 files changed, 188 insertions(+), 113 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc index d8fd612f0f..b974f78463 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc @@ -518,41 +518,38 @@ void RepeatedEnum::GenerateSerializeWithCachedSizesToArray( } void RepeatedEnum::GenerateByteSize(io::Printer* p) const { + if (has_cached_size_) { + ABSL_CHECK(field_->is_packed()); + p->Emit(R"cc( + total_size += ::_pbi::WireFormatLite::EnumSizeWithPackedTagSize( + this_._internal_$name$(), $kTagBytes$, this_.$cached_size_$); + )cc"); + return; + } p->Emit( { - {"add_to_size", + {"tag_size", [&] { - if (!field_->is_packed()) { + if (field_->is_packed()) { p->Emit(R"cc( - total_size += std::size_t{$kTagBytes$} * count; + data_size == 0 + ? 0 + : $kTagBytes$ + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)); )cc"); - return; - } - - p->Emit(R"cc( - if (data_size > 0) { - total_size += $kTagBytes$; - total_size += ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)); - } - )cc"); - if (has_cached_size_) { + } else { p->Emit(R"cc( - this_.$cached_size_$.Set(::_pbi::ToCachedSize(data_size)); + std::size_t{$kTagBytes$} * + ::_pbi::FromIntSize(this_._internal_$name$_size()); )cc"); } }}, }, R"cc( - std::size_t data_size = 0; - auto count = static_cast(this_._internal_$name$_size()); - - for (std::size_t i = 0; i < count; ++i) { - data_size += ::_pbi::WireFormatLite::EnumSize( - this_._internal_$name$().Get(static_cast(i))); - } - total_size += data_size; - $add_to_size$; + std::size_t data_size = + ::_pbi::WireFormatLite::EnumSize(this_._internal_$name$()); + std::size_t tag_size = $tag_size$; + total_size += data_size + tag_size; )cc"); } } // namespace diff --git a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc index 76a2fb2d2d..7590512e5a 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc @@ -590,54 +590,53 @@ void RepeatedPrimitive::GenerateSerializeWithCachedSizesToArray( } void RepeatedPrimitive::GenerateByteSize(io::Printer* p) const { + if (HasCachedSize()) { + ABSL_CHECK(field_->is_packed()); + p->Emit( + R"cc( + total_size += + ::_pbi::WireFormatLite::$DeclaredType$SizeWithPackedTagSize( + this_._internal_$name$(), $kTagBytes$, + this_.$_field_cached_byte_size_$); + )cc"); + return; + } p->Emit( { - Sub{"data_size", - [&] { - auto fixed_size = FixedSize(field_->type()); - if (fixed_size.has_value()) { - p->Emit({{"kFixed", *fixed_size}}, R"cc( - std::size_t{$kFixed$} * - ::_pbi::FromIntSize(this_._internal_$name$_size()) - )cc"); - } else { - p->Emit(R"cc( - ::_pbi::WireFormatLite::$DeclaredType$Size( - this_._internal_$name$()) - )cc"); - } - }} // Here and below, we need to disable the default ;-chomping - // that closure substitutions do. - .WithSuffix(""), - {"maybe_cache_data_size", + {"data_size", + [&] { + auto fixed_size = FixedSize(field_->type()); + if (fixed_size.has_value()) { + p->Emit({{"kFixed", *fixed_size}}, R"cc( + std::size_t{$kFixed$} * + ::_pbi::FromIntSize(this_._internal_$name$_size()); + )cc"); + } else { + p->Emit(R"cc( + ::_pbi::WireFormatLite::$DeclaredType$Size( + this_._internal_$name$()); + )cc"); + } + }}, + {"tag_size", [&] { - if (!HasCachedSize()) return; - p->Emit(R"cc( - this_.$_field_cached_byte_size_$.Set( - ::_pbi::ToCachedSize(data_size)); - )cc"); + if (field_->is_packed()) { + p->Emit(R"cc( + data_size == 0 + ? 0 + : $kTagBytes$ + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)); + )cc"); + } else { + p->Emit(R"cc( + std::size_t{$kTagBytes$} * + ::_pbi::FromIntSize(this_._internal_$name$_size()); + )cc"); + } }}, - Sub{"tag_size", - [&] { - if (field_->is_packed()) { - p->Emit(R"cc( - data_size == 0 - ? 0 - : $kTagBytes$ + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - )cc"); - } else { - p->Emit(R"cc( - std::size_t{$kTagBytes$} * - ::_pbi::FromIntSize(this_._internal_$name$_size()); - )cc"); - } - }} - .WithSuffix(""), }, R"cc( std::size_t data_size = $data_size$; - $maybe_cache_data_size$; std::size_t tag_size = $tag_size$; total_size += tag_size + data_size; )cc"); diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index bd9612cbea..4b3f6ce1b5 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -3304,21 +3304,17 @@ PROTOBUF_NOINLINE void FileDescriptorProto::Clear() { // repeated int32 public_dependency = 10; { std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_public_dependency()) - ; + this_._internal_public_dependency()); std::size_t tag_size = std::size_t{1} * ::_pbi::FromIntSize(this_._internal_public_dependency_size()); - ; total_size += tag_size + data_size; } // repeated int32 weak_dependency = 11; { std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_weak_dependency()) - ; + this_._internal_weak_dependency()); std::size_t tag_size = std::size_t{1} * ::_pbi::FromIntSize(this_._internal_weak_dependency_size()); - ; total_size += tag_size + data_size; } } @@ -10112,15 +10108,11 @@ PROTOBUF_NOINLINE void FieldOptions::Clear() { { // repeated .google.protobuf.FieldOptions.OptionTargetType targets = 19; { - std::size_t data_size = 0; - auto count = static_cast(this_._internal_targets_size()); - - for (std::size_t i = 0; i < count; ++i) { - data_size += ::_pbi::WireFormatLite::EnumSize( - this_._internal_targets().Get(static_cast(i))); - } - total_size += data_size; - total_size += std::size_t{2} * count; + std::size_t data_size = + ::_pbi::WireFormatLite::EnumSize(this_._internal_targets()); + std::size_t tag_size = std::size_t{2} * + ::_pbi::FromIntSize(this_._internal_targets_size()); + total_size += data_size + tag_size; } // repeated .google.protobuf.FieldOptions.EditionDefault edition_defaults = 20; { @@ -14098,31 +14090,17 @@ PROTOBUF_NOINLINE void SourceCodeInfo_Location::Clear() { { // repeated int32 path = 1 [packed = true]; { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_path()) - ; - this_._impl_._path_cached_byte_size_.Set( - ::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + total_size += + ::_pbi::WireFormatLite::Int32SizeWithPackedTagSize( + this_._internal_path(), 1, + this_._impl_._path_cached_byte_size_); } // repeated int32 span = 2 [packed = true]; { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_span()) - ; - this_._impl_._span_cached_byte_size_.Set( - ::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + total_size += + ::_pbi::WireFormatLite::Int32SizeWithPackedTagSize( + this_._internal_span(), 1, + this_._impl_._span_cached_byte_size_); } // repeated string leading_detached_comments = 6; { @@ -14692,17 +14670,10 @@ PROTOBUF_NOINLINE void GeneratedCodeInfo_Annotation::Clear() { { // repeated int32 path = 1 [packed = true]; { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_path()) - ; - this_._impl_._path_cached_byte_size_.Set( - ::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + total_size += + ::_pbi::WireFormatLite::Int32SizeWithPackedTagSize( + this_._internal_path(), 1, + this_._impl_._path_cached_byte_size_); } } cached_has_bits = this_._impl_._has_bits_[0]; diff --git a/src/google/protobuf/wire_format_lite.cc b/src/google/protobuf/wire_format_lite.cc index 8fc29af555..e20eda908c 100644 --- a/src/google/protobuf/wire_format_lite.cc +++ b/src/google/protobuf/wire_format_lite.cc @@ -21,6 +21,7 @@ #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "google/protobuf/message_lite.h" #include "utf8_validity.h" @@ -830,6 +831,91 @@ size_t WireFormatLite::SInt64Size(const RepeatedField& value) { #endif +size_t WireFormatLite::Int32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = Int32Size(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::Int64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = Int64Size(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::UInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = UInt32Size(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::UInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = UInt64Size(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::SInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = SInt32Size(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::SInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = SInt64Size(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::EnumSizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = EnumSize(value); + cached_size.Set(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/wire_format_lite.h b/src/google/protobuf/wire_format_lite.h index ad0167c850..42634885cd 100644 --- a/src/google/protobuf/wire_format_lite.h +++ b/src/google/protobuf/wire_format_lite.h @@ -700,6 +700,28 @@ class PROTOBUF_EXPORT WireFormatLite { static size_t SInt64Size(const RepeatedField& value); static size_t EnumSize(const RepeatedField& value); + static size_t Int32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t Int64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t UInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t UInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t SInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t SInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t EnumSizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + // These types always have the same size. static constexpr size_t kFixed32Size = 4; static constexpr size_t kFixed64Size = 8; From 0ac6994a46e1843daf95c41499382de311873a15 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 18 Jul 2024 11:44:31 -0700 Subject: [PATCH 024/107] Use TrivialAtomicInt for Extension::cached_size. We can't directly use std::atomic because Extension needs to be trivially copyable. Also move the Extension member functions before the data members to comply with Google C++ style guide. PiperOrigin-RevId: 653701040 --- src/google/protobuf/extension_set.cc | 6 ++-- src/google/protobuf/extension_set.h | 53 ++++++++++++++++++---------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index 21f24037a9..dee8e7795d 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -1358,7 +1358,7 @@ size_t ExtensionSet::Extension::ByteSize(int number) const { break; } - cached_size = ToCachedSize(result); + cached_size.set(ToCachedSize(result)); if (result > 0) { result += io::CodedOutputStream::VarintSize32(result); result += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag( @@ -1678,12 +1678,12 @@ uint8_t* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray( uint8_t* target, io::EpsCopyOutputStream* stream) const { if (is_repeated) { if (is_packed) { - if (cached_size == 0) return target; + if (cached_size() == 0) return target; target = stream->EnsureSpace(target); target = WireFormatLite::WriteTagToArray( number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, target); - target = WireFormatLite::WriteInt32NoTagToArray(cached_size, target); + target = WireFormatLite::WriteInt32NoTagToArray(cached_size(), target); switch (real_type(type)) { #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 8b95b72249..2d8e753029 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -632,7 +632,41 @@ class PROTOBUF_EXPORT ExtensionSet { } static std::atomic maybe_create_lazy_extension_; + + // We can't directly use std::atomic for Extension::cached_size because + // Extension needs to be trivially copyable. + class TrivialAtomicInt { + public: + int operator()() const { + return reinterpret_cast(int_)->load( + std::memory_order_relaxed); + } + void set(int v) { + reinterpret_cast(int_)->store(v, std::memory_order_relaxed); + } + + private: + using AtomicT = std::atomic; + alignas(AtomicT) char int_[sizeof(AtomicT)]; + }; + struct Extension { + // Some helper methods for operations on a single Extension. + uint8_t* InternalSerializeFieldWithCachedSizesToArray( + const MessageLite* extendee, const ExtensionSet* extension_set, + int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; + uint8_t* InternalSerializeMessageSetItemWithCachedSizesToArray( + const MessageLite* extendee, const ExtensionSet* extension_set, + int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; + size_t ByteSize(int number) const; + size_t MessageSetItemByteSize(int number) const; + void Clear(); + int GetSize() const; + void Free(); + size_t SpaceUsedExcludingSelfLong() const; + bool IsInitialized(const ExtensionSet* ext_set, const MessageLite* extendee, + int number, Arena* arena) const; + // The order of these fields packs Extension into 24 bytes when using 8 // byte alignment. Consider this when adding or removing fields here. union { @@ -683,29 +717,12 @@ class PROTOBUF_EXPORT ExtensionSet { // For packed fields, the size of the packed data is recorded here when // ByteSize() is called then used during serialization. - // TODO: Use atomic when C++ supports it. - mutable int cached_size; + mutable TrivialAtomicInt cached_size; // The descriptor for this extension, if one exists and is known. May be // nullptr. Must not be nullptr if the descriptor for the extension does // not live in the same pool as the descriptor for the containing type. const FieldDescriptor* descriptor; - - // Some helper methods for operations on a single Extension. - uint8_t* InternalSerializeFieldWithCachedSizesToArray( - const MessageLite* extendee, const ExtensionSet* extension_set, - int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; - uint8_t* InternalSerializeMessageSetItemWithCachedSizesToArray( - const MessageLite* extendee, const ExtensionSet* extension_set, - int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; - size_t ByteSize(int number) const; - size_t MessageSetItemByteSize(int number) const; - void Clear(); - int GetSize() const; - void Free(); - size_t SpaceUsedExcludingSelfLong() const; - bool IsInitialized(const ExtensionSet* ext_set, const MessageLite* extendee, - int number, Arena* arena) const; }; // The Extension struct is small enough to be passed by value, so we use it From a5aa1ba98608e16922f2f2af923fbe559bcb73c0 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 18 Jul 2024 12:51:34 -0700 Subject: [PATCH 025/107] Code simplification in RepeatedField: use Mutable to implement Set. PiperOrigin-RevId: 653723301 --- src/google/protobuf/repeated_field.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 66c2351fae..027c80e353 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -658,9 +658,7 @@ inline Element* RepeatedField::Mutable(int index) template inline void RepeatedField::Set(int index, const Element& value) { - ABSL_DCHECK_GE(index, 0); - ABSL_DCHECK_LT(index, size()); - elements()[index] = value; + *Mutable(index) = value; } template From 501fb26927952edc71dc1d721b14368f90e18156 Mon Sep 17 00:00:00 2001 From: Deanna Garcia Date: Thu, 18 Jul 2024 13:16:02 -0700 Subject: [PATCH 026/107] Add kotlin deprecation to messageOrNull extension properties in the generated code when the corresponding message is deprecated. Additionally adds tests to ensure deprecated fields are properly annotated as such. Fixes https://github.com/protocolbuffers/protobuf/issues/17084. PiperOrigin-RevId: 653731276 --- java/kotlin/BUILD.bazel | 1 + .../test/kotlin/com/google/protobuf/Proto2Test.kt | 13 +++++++++++++ src/google/protobuf/compiler/java/full/message.cc | 5 +++++ .../protobuf/compiler/java/full/message_field.cc | 1 + src/google/protobuf/compiler/java/lite/message.cc | 5 +++++ .../protobuf/compiler/java/lite/message_field.cc | 4 +++- 6 files changed, 28 insertions(+), 1 deletion(-) diff --git a/java/kotlin/BUILD.bazel b/java/kotlin/BUILD.bazel index 3af561b74d..7faced6715 100644 --- a/java/kotlin/BUILD.bazel +++ b/java/kotlin/BUILD.bazel @@ -276,6 +276,7 @@ kt_jvm_library( "//java/core:test_util", "@maven//:com_google_truth_truth", "@maven//:junit_junit", + "@rules_kotlin//kotlin/compiler:kotlin-reflect", ], ) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt index 7c319bfaac..754a3294cc 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt @@ -26,12 +26,14 @@ import protobuf_unittest.UnittestProto.TestAllTypes.NestedEnum import protobuf_unittest.UnittestProto.TestEmptyMessage import protobuf_unittest.UnittestProto.TestEmptyMessageWithExtensions import protobuf_unittest.copy +import protobuf_unittest.deprecatedMessageOrNull import protobuf_unittest.foreignMessage import protobuf_unittest.optionalGroupExtension import protobuf_unittest.optionalNestedMessageOrNull import protobuf_unittest.repeatedGroupExtension import protobuf_unittest.testAllExtensions import protobuf_unittest.testAllTypes +import protobuf_unittest.testDeprecatedFields import protobuf_unittest.testEmptyMessage import protobuf_unittest.testEmptyMessageWithExtensions import protobuf_unittest.testEnumMap @@ -959,4 +961,15 @@ class Proto2Test { assertThat(someNestedMessage.optionalNestedMessageOrNull) .isEqualTo(TestAllTypesKt.nestedMessage { bb = 118 }) } + + @Test + fun testDeprecated() { + val testInstance = + protobuf_unittest.UnittestProto.TestDeprecatedFields.getDefaultInstance() + assertThat(testInstance::deprecatedMessageOrNull.annotations.any { it is Deprecated }).isTrue() + + val unused = testDeprecatedFields { + assertThat(::deprecatedMessage.annotations.any { it is Deprecated }).isTrue() + } + } } diff --git a/src/google/protobuf/compiler/java/full/message.cc b/src/google/protobuf/compiler/java/full/message.cc index 433177312a..9627be0ae2 100644 --- a/src/google/protobuf/compiler/java/full/message.cc +++ b/src/google/protobuf/compiler/java/full/message.cc @@ -1330,6 +1330,11 @@ void ImmutableMessageGenerator::GenerateKotlinOrNull( for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); if (field->has_presence() && GetJavaType(field) == JAVATYPE_MESSAGE) { + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } printer->Print( "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " "$full_name$?\n" diff --git a/src/google/protobuf/compiler/java/full/message_field.cc b/src/google/protobuf/compiler/java/full/message_field.cc index 54ae8dea17..f26c57b072 100644 --- a/src/google/protobuf/compiler/java/full/message_field.cc +++ b/src/google/protobuf/compiler/java/full/message_field.cc @@ -409,6 +409,7 @@ void ImmutableMessageFieldGenerator::GenerateKotlinOrNull(io::Printer* printer) if (descriptor_->has_presence() && descriptor_->real_containing_oneof() == nullptr) { printer->Print(variables_, + "$kt_deprecation$\n" "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" " get() = $kt_dsl_builder$.$name$OrNull\n"); } diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index 39222bbb9b..f043476aa5 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -854,6 +854,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); if (field->has_presence() && GetJavaType(field) == JAVATYPE_MESSAGE) { + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } printer->Print( "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " "$full_name$?\n" diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index 84a10b9888..0eb6340729 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -309,10 +309,12 @@ void ImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( GenerateKotlinOrNull(printer); } -void ImmutableMessageFieldLiteGenerator::GenerateKotlinOrNull(io::Printer* printer) const { +void ImmutableMessageFieldLiteGenerator::GenerateKotlinOrNull( + io::Printer* printer) const { if (descriptor_->has_presence() && descriptor_->real_containing_oneof() == nullptr) { printer->Print(variables_, + "$kt_deprecation$\n" "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" " get() = $kt_dsl_builder$.$name$OrNull\n"); } From b764fd9a195c99a2dcc1530a2344a08c65a7dd16 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Jul 2024 13:27:49 -0700 Subject: [PATCH 027/107] Upgrade jsoncpp to 1.9.5 PiperOrigin-RevId: 653735407 --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index eea6072696..97f95e3ac4 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -84,8 +84,8 @@ def protobuf_deps(): _github_archive( name = "jsoncpp", repo = "https://github.com/open-source-parsers/jsoncpp", - commit = "9059f5cad030ba11d37818847443a53918c327b1", # 1.9.4 - sha256 = "c0c583c7b53a53bcd1f7385f15439dcdf0314d550362379e2db9919a918d1996", + commit = "5defb4ed1a4293b8e2bf641e16b156fb9de498cc", # 1.9.5 + sha256 = "a03d3136ff6dd092143bba8d3ded641e87b44e6c0b1f632b368f6cc8587524b5", build_file = Label("//:third_party/jsoncpp.BUILD"), ) From 8995b485b94cad63e5b02841e6c9d0bed4c2ffc2 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Thu, 18 Jul 2024 14:09:32 -0700 Subject: [PATCH 028/107] Parameterized the Kotlin DSL code generator so that it can emit output that works on non-JVM platforms. Note however that we do not currently have open-source support non-JVM platforms for the main (non-DSL) Protobuf library. PiperOrigin-RevId: 653751253 --- src/google/protobuf/compiler/java/BUILD.bazel | 1 + src/google/protobuf/compiler/java/context.cc | 1 + .../protobuf/compiler/java/field_common.cc | 1 + .../protobuf/compiler/java/field_common.h | 2 + src/google/protobuf/compiler/java/helpers.h | 17 ++ .../compiler/java/kotlin_generator.cc | 7 +- .../protobuf/compiler/java/lite/enum_field.cc | 167 ++++++++++------- .../protobuf/compiler/java/lite/map_field.cc | 65 ++++--- .../protobuf/compiler/java/lite/message.cc | 168 ++++++++++++------ .../compiler/java/lite/message_field.cc | 163 ++++++++++------- .../compiler/java/lite/primitive_field.cc | 167 ++++++++++------- .../compiler/java/lite/string_field.cc | 129 ++++++++------ src/google/protobuf/compiler/java/options.h | 8 +- 13 files changed, 558 insertions(+), 338 deletions(-) diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index f188e07ca8..3aca276057 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -197,6 +197,7 @@ cc_library( "//src/google/protobuf", "//src/google/protobuf:port", "//src/google/protobuf/compiler:code_generator", + "@com_google_absl//absl/strings", ], ) diff --git a/src/google/protobuf/compiler/java/context.cc b/src/google/protobuf/compiler/java/context.cc index e37afbcb18..1469b59a98 100644 --- a/src/google/protobuf/compiler/java/context.cc +++ b/src/google/protobuf/compiler/java/context.cc @@ -153,6 +153,7 @@ void Context::InitializeFieldGeneratorInfoForFields( absl::StrAppend(&info.capitalized_name, field->number()); info.disambiguated_reason = conflict_reason[i]; } + info.options = options_; field_generator_info_map_[field] = info; } } diff --git a/src/google/protobuf/compiler/java/field_common.cc b/src/google/protobuf/compiler/java/field_common.cc index d8d7255e7b..7ff7f0cd48 100644 --- a/src/google/protobuf/compiler/java/field_common.cc +++ b/src/google/protobuf/compiler/java/field_common.cc @@ -41,6 +41,7 @@ void SetCommonFieldVariables( (*variables)["kt_capitalized_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->capitalized_name, "_") : info->capitalized_name; + (*variables)["jvm_synthetic"] = JvmSynthetic(info->options); if (!descriptor->is_repeated()) { (*variables)["annotation_field_type"] = std::string(FieldTypeName(descriptor->type())); diff --git a/src/google/protobuf/compiler/java/field_common.h b/src/google/protobuf/compiler/java/field_common.h index c29d1c25e9..1f4602634c 100644 --- a/src/google/protobuf/compiler/java/field_common.h +++ b/src/google/protobuf/compiler/java/field_common.h @@ -3,6 +3,7 @@ #include +#include "google/protobuf/compiler/java/options.h" #include "google/protobuf/descriptor.h" namespace google { @@ -15,6 +16,7 @@ struct FieldGeneratorInfo { std::string name; std::string capitalized_name; std::string disambiguated_reason; + Options options; }; // Oneof information used in OneofFieldGenerators. diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index 3cd40e2d0e..fa359f9120 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -380,6 +380,23 @@ const FieldDescriptor* MapKeyField(const FieldDescriptor* descriptor); const FieldDescriptor* MapValueField(const FieldDescriptor* descriptor); +inline std::string JvmSynthetic(const Options& options) { + return options.jvm_dsl ? "@kotlin.jvm.JvmSynthetic\n" : ""; +} + +struct JvmNameContext { + const Options& options; + io::Printer* printer; +}; + +inline void JvmName(absl::string_view name, const JvmNameContext& context) { + if (!context.options.jvm_dsl) return; + context.printer->Emit("@kotlin.jvm.JvmName(\""); + // Note: `name` will likely have vars in it that we do want to interpolate. + context.printer->Emit(name); + context.printer->Emit("\")\n"); +} + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/kotlin_generator.cc b/src/google/protobuf/compiler/java/kotlin_generator.cc index c9848b390d..9c44c7acf1 100644 --- a/src/google/protobuf/compiler/java/kotlin_generator.cc +++ b/src/google/protobuf/compiler/java/kotlin_generator.cc @@ -7,6 +7,7 @@ #include "google/protobuf/compiler/java/kotlin_generator.h" +#include "absl/strings/str_cat.h" #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/java/file.h" #include "google/protobuf/compiler/java/generator.h" @@ -57,6 +58,8 @@ bool KotlinGenerator::Generate(const FileDescriptor* file, file_options.annotation_list_file = option.second; } else if (option.first == "experimental_strip_nonfunctional_codegen") { file_options.strip_nonfunctional_codegen = true; + } else if (option.first == "no_jvm_dsl") { + file_options.jvm_dsl = false; } else { *error = absl::StrCat("Unknown generator option: ", option.first); return false; @@ -81,8 +84,8 @@ bool KotlinGenerator::Generate(const FileDescriptor* file, return std::unique_ptr(context->Open(filename)); }; std::string package_dir = JavaPackageToDir(file_generator->java_package()); - std::string kotlin_filename = - absl::StrCat(package_dir, file_generator->GetKotlinClassname(), ".kt"); + std::string kotlin_filename = absl::StrCat( + package_dir, file_generator->GetKotlinClassname(), ".proto.kt"); all_files.push_back(kotlin_filename); std::string info_full_path = absl::StrCat(kotlin_filename, ".pb.meta"); if (file_options.annotate_code) { diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index bffec06347..db2d5ef658 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -303,24 +303,37 @@ void ImmutableEnumFieldLiteGenerator::GenerateBuilderMembers( void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); if (SupportUnknownEnumValue(descriptor_)) { - printer->Print( - variables_, + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$Value$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$Value$}$", name_ctx); }}, + }, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" - " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" + " $jvm_name_get$" " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" + " $jvm_name_set$" " set(value) {\n" " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" " }\n"); @@ -329,17 +342,16 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -894,8 +906,9 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateInitializationCode( void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -907,59 +920,73 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$ public val $kt_name$: " - "com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" - " )\n"); + printer->Print( + "$kt_deprecation$ public val $kt_name$: " + "com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" + "$ jvm_synthetic$" + " get() = com.google.protobuf.kotlin.DslList(\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "add(value: $kt_type$) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "add(value: $kt_type$) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "plusAssign(value: $kt_type$) {\n" - " add(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "plusAssign(value: $kt_type$) {\n" + " add(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" - " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" + " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." @@ -970,10 +997,12 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." "set(index: kotlin.Int, value: $kt_type$) {\n" @@ -983,14 +1012,18 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}"); } std::string RepeatedImmutableEnumFieldLiteGenerator::GetBoxedType() const { diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 7ab58c50ba..29d285c2e3 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -837,8 +837,9 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -850,22 +851,27 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, + printer->Emit( + { + {"jvm_name", + [&] { JvmName("get$kt_capitalized_name$Map", name_ctx); }}, + }, "$kt_deprecation$ public val $kt_name$: " "com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " @JvmName(\"get$kt_capitalized_name$Map\")\n" + "$ jvm_synthetic$" + "$jvm_name$" " get() = com.google.protobuf.kotlin.DslMap(\n" " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@JvmName(\"put$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("put$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .put(key: $kt_key_type$, value: $kt_value_type$) {\n" @@ -874,10 +880,12 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" @@ -887,10 +895,13 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"remove$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("remove$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .remove(key: $kt_key_type$) {\n" @@ -899,10 +910,13 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"putAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("putAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .putAll(map: kotlin.collections.Map<$kt_key_type$, $kt_value_type$>) " @@ -912,10 +926,13 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"clear$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .clear() {\n" diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index f043476aa5..236698c1b1 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -751,16 +751,16 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( " private val _builder: $message$.Builder\n" ") {\n" " public companion object {\n" - " @kotlin.jvm.JvmSynthetic\n" + "$ jvm_synthetic$" " @kotlin.PublishedApi\n" " internal fun _create(builder: $message$.Builder): Dsl = " "Dsl(builder)\n" " }\n" "\n" - " @kotlin.jvm.JvmSynthetic\n" + "$ jvm_synthetic$" " @kotlin.PublishedApi\n" " internal fun _build(): $message$ = _builder.build()\n", - "message", + "jvm_synthetic", JvmSynthetic(context_->options()), "message", EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true))); printer->Indent(); @@ -772,19 +772,26 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( } for (auto& kv : oneofs_) { + JvmNameContext name_ctx = {context_->options(), printer}; const OneofDescriptor* oneof = kv.second; auto oneof_name = context_->GetOneofGeneratorInfo(oneof)->name; - printer->Print( + printer->Emit( + { + {"jvm_name", + [&] { JvmName("get$oneof_capitalized_name$Case", name_ctx); }}, + {"oneof_name", oneof_name}, + {"oneof_capitalized_name", + context_->GetOneofGeneratorInfo(oneof)->capitalized_name}, + {"oneof_property_name", GetKotlinPropertyName(oneof_name)}, + {"message", EscapeKotlinKeywords( + name_resolver_->GetClassName(descriptor_, true))}, + }, "public val $oneof_name$Case: $message$.$oneof_capitalized_name$Case\n" - " @JvmName(\"get$oneof_capitalized_name$Case\")\n" + "$jvm_name$" " get() = _builder.$oneof_property_name$Case\n\n" "public fun clear$oneof_capitalized_name$() {\n" " _builder.clear$oneof_capitalized_name$()\n" - "}\n", - "oneof_name", oneof_name, "oneof_capitalized_name", - context_->GetOneofGeneratorInfo(oneof)->capitalized_name, - "oneof_property_name", GetKotlinPropertyName(oneof_name), "message", - EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true))); + "}\n"); } if (descriptor_->extension_range_count() > 0) { @@ -797,9 +804,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( void ImmutableMessageLiteGenerator::GenerateKotlinMembers( io::Printer* printer) const { - printer->Print("@kotlin.jvm.JvmName(\"-initialize$camelcase_name$\")\n", - "camelcase_name", - name_resolver_->GetKotlinFactoryName(descriptor_)); + if (context_->options().jvm_dsl) { + printer->Print("@kotlin.jvm.JvmName(\"-initialize$camelcase_name$\")\n", + "camelcase_name", + name_resolver_->GetKotlinFactoryName(descriptor_)); + } printer->Print( "public inline fun $camelcase_name$(block: $message_kt$.Dsl.() -> " @@ -850,7 +859,9 @@ void ImmutableMessageLiteGenerator::GenerateTopLevelKotlinMembers( void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( io::Printer* printer) const { - // Generate getFieldOrNull getters for all optional message fields. + // Generate getFieldOrNull getters for all optional message fields in both + // `Foo` and `Foo.Builder`. We don't simply generate them in `FooOrBuilder` + // because some Kotlin proto implementations don't have `FooOrBuilder`. for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); if (field->has_presence() && GetJavaType(field) == JAVATYPE_MESSAGE) { @@ -860,28 +871,46 @@ void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( "name", context_->GetFieldGeneratorInfo(field)->name); } printer->Print( - "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " + "public val $full_classname$.$camelcase_name$OrNull: " "$full_name$?\n" - " get() = if (has$name$()) get$name$() else null\n\n", + " get() = if (has$capitalized_name$()) this.$name$ else null\n\n", "full_classname", EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)), "camelcase_name", context_->GetFieldGeneratorInfo(field)->name, "full_name", EscapeKotlinKeywords( name_resolver_->GetImmutableClassName(field->message_type())), - "name", context_->GetFieldGeneratorInfo(field)->capitalized_name); + "capitalized_name", + context_->GetFieldGeneratorInfo(field)->capitalized_name, "name", + EscapeKotlinKeywords(GetKotlinPropertyName( + context_->GetFieldGeneratorInfo(field)->capitalized_name))); + printer->Print( + "public val $full_classname$.Builder.$camelcase_name$OrNull: " + "$full_name$?\n" + " get() = if (has$capitalized_name$()) this.$name$ else null\n\n", + "full_classname", + EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)), + "camelcase_name", context_->GetFieldGeneratorInfo(field)->name, + "full_name", + EscapeKotlinKeywords( + name_resolver_->GetImmutableClassName(field->message_type())), + "capitalized_name", + context_->GetFieldGeneratorInfo(field)->capitalized_name, "name", + EscapeKotlinKeywords(GetKotlinPropertyName( + context_->GetFieldGeneratorInfo(field)->capitalized_name))); } } } void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( io::Printer* printer) const { + JvmNameContext name_ctx = {context_->options(), printer}; std::string message_name = EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)); printer->Print( "@Suppress(\"UNCHECKED_CAST\")\n" - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public operator fun get(extension: " "com.google.protobuf.ExtensionLite<$message$, T>): T {\n" " return if (extension.isRepeated) {\n" @@ -891,51 +920,63 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( " _builder.getExtension(extension)\n" " }\n" "}\n\n", - "message", message_name); - - printer->Print( - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.OptIn" - "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" - "@kotlin.jvm.JvmName(\"-getRepeatedExtension\")\n" - "public operator fun get(\n" - " extension: com.google.protobuf.ExtensionLite<$message$, " - "kotlin.collections.List>\n" - "): com.google.protobuf.kotlin.ExtensionList {\n" - " return com.google.protobuf.kotlin.ExtensionList(extension, " - "_builder.getExtension(extension))\n" - "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); + + if (context_->options().jvm_dsl) { + // TODO: generate this on Kotlin Native once the [Mutable]List + // issue is resolved. + printer->Emit( + { + {"jvm_name", [&] { JvmName("-getRepeatedExtension", name_ctx); }}, + {"jvm_synthetic", JvmSynthetic(context_->options())}, + {"message", message_name}, + }, + "$jvm_synthetic$" + "@kotlin.OptIn" + "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" + "$jvm_name$" + "public operator fun get(\n" + " extension: com.google.protobuf.ExtensionLite<$message$, " + "kotlin.collections.List>\n" + "): com.google.protobuf.kotlin.ExtensionList {\n" + " return com.google.protobuf.kotlin.ExtensionList(extension, " + "_builder.getExtension(extension))\n" + "}\n\n"); + } printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public operator fun contains(extension: " "com.google.protobuf.ExtensionLite<$message$, *>): " "Boolean {\n" " return _builder.hasExtension(extension)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun clear(extension: " "com.google.protobuf.ExtensionLite<$message$, *>) " "{\n" " _builder.clearExtension(extension)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun setExtension(extension: " "com.google.protobuf.ExtensionLite<$message$, T>, " "value: T) {\n" " _builder.setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun > set(\n" " extension: com.google.protobuf.ExtensionLite<$message$, T>,\n" @@ -943,10 +984,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( ") {\n" " setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun set(\n" " extension: com.google.protobuf.ExtensionLite<$message$, " @@ -955,10 +997,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( ") {\n" " setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun set(\n" " extension: com.google.protobuf.ExtensionLite<$message$, T>,\n" @@ -966,18 +1009,24 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( ") {\n" " setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); + + // TODO: generate these methods on Kotlin Native once the + // [Mutable]List issue is resolved. + if (!context_->options().jvm_dsl) return; printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun com.google.protobuf.kotlin.ExtensionList.add(value: E) {\n" " _builder.addExtension(this.extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun " "com.google.protobuf.kotlin.ExtensionListoptions()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun com.google.protobuf.kotlin.ExtensionList.addAll(values: Iterable) {\n" " for (value in values) {\n" " add(value)\n" " }\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun " "com.google.protobuf.kotlin.ExtensionList) {\n" " addAll(values)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public operator fun " "com.google.protobuf.kotlin.ExtensionList.set(index: Int, value: " "E) {\n" " _builder.setExtension(this.extension, index, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline fun com.google.protobuf.kotlin.ExtensionList<*, " "$message$>.clear() {\n" " clear(extension)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); } } // namespace java diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index 0eb6340729..d719357318 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -279,30 +279,37 @@ void ImmutableMessageFieldLiteGenerator::GenerateBuilderMembers( void ImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -313,10 +320,10 @@ void ImmutableMessageFieldLiteGenerator::GenerateKotlinOrNull( io::Printer* printer) const { if (descriptor_->has_presence() && descriptor_->real_containing_oneof() == nullptr) { - printer->Print(variables_, - "$kt_deprecation$\n" - "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" - " get() = $kt_dsl_builder$.$name$OrNull\n"); + printer->Print( + "$kt_deprecation$\n" + "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" + " get() = $kt_dsl_builder$.$name$OrNull\n"); } } @@ -802,8 +809,9 @@ std::string RepeatedImmutableMessageFieldLiteGenerator::GetBoxedType() const { void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -815,59 +823,73 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$ public val $kt_name$: " - "com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" - " )\n"); + printer->Print( + "$kt_deprecation$ public val $kt_name$: " + "com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" + "$ jvm_synthetic$" + " get() = com.google.protobuf.kotlin.DslList(\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "add(value: $kt_type$) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "add(value: $kt_type$) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "plusAssign(value: $kt_type$) {\n" - " add(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "plusAssign(value: $kt_type$) {\n" + " add(value)\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" - " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" + " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." @@ -878,10 +900,12 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." "set(index: kotlin.Int, value: $kt_type$) {\n" @@ -891,14 +915,19 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); } } // namespace java diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index 838df002ce..3e0d174671 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -328,43 +328,55 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void ImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options()); if (descriptor_->name() == "is_initialized") { - printer->Print( - variables_, + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, "// TODO: b/336400327 - remove this hack; we should access properties\n" "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " $jvm_name_get$" " get() = $kt_dsl_builder$.get${$$kt_capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " $jvm_name_set$" " set(value) {\n" " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" " }\n"); } else { - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); } WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -713,8 +725,9 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -725,59 +738,73 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( " : com.google.protobuf.kotlin.DslProxy()\n"); WriteFieldDocComment(printer, descriptor_, context_->options()); - printer->Print(variables_, - "$kt_deprecation$ public val $kt_name$: " - "com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" - " )\n"); + printer->Print( + "$kt_deprecation$ public val $kt_name$: " + "com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" + "$ jvm_synthetic$" + " get() = com.google.protobuf.kotlin.DslList(\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "add(value: $kt_type$) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "add(value: $kt_type$) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "plusAssign(value: $kt_type$) {\n" - " add(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "plusAssign(value: $kt_type$) {\n" + " add(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" - " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" + " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." @@ -788,10 +815,12 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." "set(index: kotlin.Int, value: $kt_type$) {\n" @@ -801,14 +830,18 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}"); } void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateFieldInfo( diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index 5e926422bc..e3717b3245 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -332,31 +332,38 @@ void ImmutableStringFieldLiteGenerator::GenerateBuilderMembers( void ImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: kotlin.String\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: kotlin.String\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -803,8 +810,9 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateBuilderMembers( void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -819,7 +827,6 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "$kt_deprecation$public val $kt_name$: " "com.google.protobuf.kotlin.DslList" "\n" @@ -833,37 +840,47 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "." - "add(value: kotlin.String) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "." + "add(value: kotlin.String) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}\n"); // List += String WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "." - "plusAssign(value: kotlin.String) {\n" - " add(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "." + "plusAssign(value: kotlin.String) {\n" + " add(value)\n" + "}\n"); // List.addAll(Iterable) WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslList" "." "addAll(values: kotlin.collections.Iterable) {\n" @@ -874,10 +891,13 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "." @@ -889,10 +909,12 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "." "set(index: kotlin.Int, value: kotlin.String) {\n" @@ -902,14 +924,17 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}"); } void RepeatedImmutableStringFieldLiteGenerator::GenerateFieldInfo( diff --git a/src/google/protobuf/compiler/java/options.h b/src/google/protobuf/compiler/java/options.h index fbf116bdc6..0328af4c9c 100644 --- a/src/google/protobuf/compiler/java/options.h +++ b/src/google/protobuf/compiler/java/options.h @@ -25,8 +25,8 @@ struct Options { generate_shared_code(false), enforce_lite(false), annotate_code(false), - strip_nonfunctional_codegen(false) { - } + strip_nonfunctional_codegen(false), + jvm_dsl(true) {} bool generate_immutable_code; bool generate_mutable_code; @@ -46,6 +46,10 @@ struct Options { std::string output_list_file; // If true, strip out nonfunctional codegen. bool strip_nonfunctional_codegen; + + // If true, generate JVM-specific DSL code. This defaults to true for + // compatibility with the old behavior. + bool jvm_dsl; }; } // namespace java From 1f005c579fb1db964530cf537befad4bd83c8c44 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Fri, 19 Jul 2024 00:34:05 -0700 Subject: [PATCH 029/107] Avoid allocations in FieldSet.mergeFromField - Pre-size ArrayList - Avoid iterator allocation PiperOrigin-RevId: 653905397 --- .../main/java/com/google/protobuf/FieldSet.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/FieldSet.java b/java/core/src/main/java/com/google/protobuf/FieldSet.java index 9ddf548865..debad4e971 100644 --- a/java/core/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/FieldSet.java @@ -517,7 +517,8 @@ final class FieldSet> { } } - @SuppressWarnings({"unchecked", "rawtypes"}) + // Avoid iterator allocation. + @SuppressWarnings({"unchecked", "ForeachList", "ForeachListWithUserVar"}) private void mergeFromField(final Map.Entry entry) { final T descriptor = entry.getKey(); Object otherValue = entry.getValue(); @@ -528,11 +529,16 @@ final class FieldSet> { throw new IllegalStateException("Lazy fields can not be repeated"); } Object value = getField(descriptor); + List otherList = (List) otherValue; + int otherListSize = otherList.size(); if (value == null) { - value = new ArrayList<>(); + value = new ArrayList<>(otherListSize); } - for (Object element : (List) otherValue) { - ((List) value).add(cloneIfMutable(element)); + List list = (List) value; + // Avoid iterator allocation. + for (int i = 0; i < otherListSize; i++) { + Object element = otherList.get(i); + list.add(cloneIfMutable(element)); } fields.put(descriptor, value); } else if (descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE) { From 15bd43bff73a46d916b5812bfacc76b3d0d9f156 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Jul 2024 07:35:12 -0700 Subject: [PATCH 030/107] Internal changes to versions PiperOrigin-RevId: 654002987 --- editions/golden/compare_cpp_codegen_failure.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editions/golden/compare_cpp_codegen_failure.xml b/editions/golden/compare_cpp_codegen_failure.xml index 05caafb727..a432b5a011 100644 --- a/editions/golden/compare_cpp_codegen_failure.xml +++ b/editions/golden/compare_cpp_codegen_failure.xml @@ -2,10 +2,10 @@ - + - + From 8d7f50f7c205a81421a08debd4bb16eabd3a04e5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Jul 2024 08:06:57 -0700 Subject: [PATCH 031/107] Automated rollback of commit d0e49dfe3161714046a4ee2aeafba133b171e26f. PiperOrigin-RevId: 654010326 --- src/google/protobuf/descriptor.cc | 23 ------- src/google/protobuf/descriptor.h | 19 +++--- src/google/protobuf/descriptor_lite.h | 11 ---- src/google/protobuf/descriptor_unittest.cc | 70 ---------------------- 4 files changed, 8 insertions(+), 115 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 4561431797..ed5f6af920 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3939,29 +3939,6 @@ bool FieldDescriptor::has_optional_keyword() const { is_optional() && !containing_oneof()); } -FieldDescriptor::CppStringType FieldDescriptor::cpp_string_type() const { - ABSL_DCHECK(cpp_type() == FieldDescriptor::CPPTYPE_STRING); - switch (features().GetExtension(pb::cpp).string_type()) { - case pb::CppFeatures::VIEW: - return CppStringType::kView; - case pb::CppFeatures::CORD: - // In open-source, protobuf CORD is only supported for singular bytes - // fields. - if (type() != FieldDescriptor::TYPE_BYTES || is_repeated() || - is_extension()) { - return CppStringType::kString; - } - return CppStringType::kCord; - case pb::CppFeatures::STRING: - return CppStringType::kString; - default: - // If features haven't been resolved, this is a dynamic build not for C++ - // codegen. Just use string type. - ABSL_DCHECK(!features().GetExtension(pb::cpp).has_string_type()); - return CppStringType::kString; - } -} - // Location methods =============================================== bool FileDescriptor::GetSourceLocation(const std::vector& path, diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 08a3d5c510..28db34d95e 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -875,10 +875,6 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase, const char* cpp_type_name() const; // Name of the C++ type. Label label() const; // optional/required/repeated -#ifndef SWIG - CppStringType cpp_string_type() const; // The C++ string type of this field. -#endif - bool is_required() const; // shorthand for label() == LABEL_REQUIRED bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL bool is_repeated() const; // shorthand for label() == LABEL_REPEATED @@ -2936,21 +2932,22 @@ PROTOBUF_EXPORT bool HasPreservingUnknownEnumSemantics( PROTOBUF_EXPORT bool HasHasbit(const FieldDescriptor* field); -#ifndef SWIG // For a string field, returns the effective ctype. If the actual ctype is // not supported, returns the default of STRING. template typename FieldOpts::CType EffectiveStringCType(const FieldDesc* field) { - // TODO Replace this function with FieldDescriptor::string_type; - switch (field->cpp_string_type()) { - case FieldDescriptor::CppStringType::kCord: - return FieldOpts::CORD; - default: - return FieldOpts::STRING; + ABSL_DCHECK(field->cpp_type() == FieldDescriptor::CPPTYPE_STRING); + // Open-source protobuf release only supports STRING ctype and CORD for + // sinuglar bytes. + if (field->type() == FieldDescriptor::TYPE_BYTES && !field->is_repeated() && + field->options().ctype() == FieldOpts::CORD && !field->is_extension()) { + return FieldOpts::CORD; } + return FieldOpts::STRING; } +#ifndef SWIG enum class Utf8CheckMode : uint8_t { kStrict = 0, // Parsing will fail if non UTF-8 data is in string fields. kVerify = 1, // Only log an error but parsing will succeed. diff --git a/src/google/protobuf/descriptor_lite.h b/src/google/protobuf/descriptor_lite.h index e1a90b7fa6..db5805affe 100644 --- a/src/google/protobuf/descriptor_lite.h +++ b/src/google/protobuf/descriptor_lite.h @@ -78,17 +78,6 @@ class FieldDescriptorLite { MAX_LABEL = 3, // Constant useful for defining lookup tables // indexed by Label. }; - - // Identifies the storage type of a C++ string field. This corresponds to - // pb.CppFeatures.StringType, but is compatible with ctype prior to Edition - // 2024. 0 is reserved for errors. -#ifndef SWIG - enum class CppStringType { - kView = 1, - kCord = 2, - kString = 3, - }; -#endif }; } // namespace internal diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index f4d3137099..0c13b3a2b4 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -7594,9 +7594,6 @@ TEST_F(FeaturesTest, Proto2Features) { EXPECT_TRUE(message->FindFieldByName("req")->is_required()); EXPECT_TRUE(file->enum_type(0)->is_closed()); - EXPECT_EQ(message->FindFieldByName("str")->cpp_string_type(), - FieldDescriptor::CppStringType::kString); - // Check round-trip consistency. FileDescriptorProto proto; file->CopyTo(&proto); @@ -9713,73 +9710,6 @@ TEST_F(FeaturesTest, EnumFeatureHelpers) { EXPECT_FALSE(HasPreservingUnknownEnumSemantics(field_legacy_closed)); } -TEST_F(FeaturesTest, FieldCppStringType) { - BuildDescriptorMessagesInTestPool(); - const std::string file_contents = absl::Substitute( - R"pb( - name: "foo.proto" - syntax: "editions" - edition: EDITION_2024 - message_type { - name: "Foo" - field { - name: "view" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "str" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - options { - features { - [pb.cpp] { string_type: STRING } - } - } - } - field { - name: "cord" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_STRING - options { - features { - [pb.cpp] { string_type: CORD } - } - } - } - field { - name: "cord_bytes" - number: 4 - label: LABEL_OPTIONAL - type: TYPE_BYTES - options { - features { - [pb.cpp] { string_type: CORD } - } - } - } $0 - } - )pb", - "" - ); - const FileDescriptor* file = BuildFile(file_contents); - const Descriptor* message = file->message_type(0); - const FieldDescriptor* view = message->field(0); - const FieldDescriptor* str = message->field(1); - const FieldDescriptor* cord = message->field(2); - const FieldDescriptor* cord_bytes = message->field(3); - - EXPECT_EQ(view->cpp_string_type(), FieldDescriptor::CppStringType::kView); - EXPECT_EQ(str->cpp_string_type(), FieldDescriptor::CppStringType::kString); - EXPECT_EQ(cord_bytes->cpp_string_type(), - FieldDescriptor::CppStringType::kCord); - EXPECT_EQ(cord->cpp_string_type(), FieldDescriptor::CppStringType::kString); - -} - TEST_F(FeaturesTest, MergeFeatureValidationFailed) { BuildDescriptorMessagesInTestPool(); BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); From 5d8f7eed1eed480a3f4a679d95820df6e7474840 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Fri, 19 Jul 2024 09:25:55 -0700 Subject: [PATCH 032/107] Address code size regression in the Kotlin DSL by special-casing the code to use FooOrBuilder on JVM only. PiperOrigin-RevId: 654029288 --- .../protobuf/compiler/java/lite/message.cc | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index 236698c1b1..6667e7aed7 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -859,12 +859,33 @@ void ImmutableMessageLiteGenerator::GenerateTopLevelKotlinMembers( void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( io::Printer* printer) const { - // Generate getFieldOrNull getters for all optional message fields in both - // `Foo` and `Foo.Builder`. We don't simply generate them in `FooOrBuilder` - // because some Kotlin proto implementations don't have `FooOrBuilder`. for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); - if (field->has_presence() && GetJavaType(field) == JAVATYPE_MESSAGE) { + if (!field->has_presence() || GetJavaType(field) != JAVATYPE_MESSAGE) { + continue; + } + if (context_->options().jvm_dsl) { + // On the JVM, we can use `FooOrBuilder`, and it saves code size to + // generate only one method instead of two. + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } + printer->Print( + "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " + "$full_name$?\n" + " get() = if (has$name$()) get$name$() else null\n\n", + "full_classname", + EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)), + "camelcase_name", context_->GetFieldGeneratorInfo(field)->name, + "full_name", + EscapeKotlinKeywords( + name_resolver_->GetImmutableClassName(field->message_type())), + "name", context_->GetFieldGeneratorInfo(field)->capitalized_name); + } else { + // Non-JVM platforms don't have `FooOrBuilder`, so we generate `Foo` + // and `Foo.Builder` methods. if (field->options().deprecated()) { printer->Print( "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", @@ -884,6 +905,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( context_->GetFieldGeneratorInfo(field)->capitalized_name, "name", EscapeKotlinKeywords(GetKotlinPropertyName( context_->GetFieldGeneratorInfo(field)->capitalized_name))); + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } printer->Print( "public val $full_classname$.Builder.$camelcase_name$OrNull: " "$full_name$?\n" From 9a0c8f6113a39a5361f8adf156ae21a8ddcc167c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Jul 2024 10:00:37 -0700 Subject: [PATCH 033/107] Inline the stub functions in custom vtable mode. That way we avoid the extra function call and they behave more like normal virtual functions. Has no effect in normal mode. PiperOrigin-RevId: 654039245 --- src/google/protobuf/message_lite.cc | 19 ++----------------- src/google/protobuf/message_lite.h | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 87ed3973f9..9abbf76576 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -64,23 +64,6 @@ void MessageLite::CheckTypeAndMergeFrom(const MessageLite& other) { data->merge_to_from(*this, other); } -#if defined(PROTOBUF_CUSTOM_VTABLE) -uint8_t* MessageLite::_InternalSerialize( - uint8_t* ptr, io::EpsCopyOutputStream* stream) const { - return _class_data_->serialize(*this, ptr, stream); -} - -MessageLite* MessageLite::New(Arena* arena) const { - return static_cast(_class_data_->new_message(this, arena)); -} - -void MessageLite::Clear() { _class_data_->clear(*this); } - -size_t MessageLite::ByteSizeLong() const { - return _class_data_->byte_size_long(*this); -} -#endif // PROTOBUF_CUSTOM_VTABLE - bool MessageLite::IsInitialized() const { auto* data = GetClassData(); return data->is_initialized != nullptr ? data->is_initialized(*this) : true; @@ -137,7 +120,9 @@ std::string MessageLite::DebugString() const { return absl::StrCat("MessageLite at 0x", absl::Hex(this)); } +#if !defined(PROTOBUF_CUSTOM_VTABLE) int MessageLite::GetCachedSize() const { return AccessCachedSize().Get(); } +#endif namespace { diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index ba04789a2b..c7dfe48e4b 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -229,7 +229,9 @@ class PROTOBUF_EXPORT MessageLite { // Construct a new instance on the arena. Ownership is passed to the caller // if arena is a nullptr. #if defined(PROTOBUF_CUSTOM_VTABLE) - MessageLite* New(Arena* arena) const; + MessageLite* New(Arena* arena) const { + return static_cast(_class_data_->new_message(this, arena)); + } #else virtual MessageLite* New(Arena* arena) const = 0; #endif // PROTOBUF_CUSTOM_VTABLE @@ -246,7 +248,7 @@ class PROTOBUF_EXPORT MessageLite { // will likely be needed again, so the memory used may not be freed. // To ensure that all memory used by a Message is freed, you must delete it. #if defined(PROTOBUF_CUSTOM_VTABLE) - void Clear(); + void Clear() { _class_data_->clear(*this); } #else virtual void Clear() = 0; #endif // PROTOBUF_CUSTOM_VTABLE @@ -470,7 +472,7 @@ class PROTOBUF_EXPORT MessageLite { // ByteSizeLong() is generally linear in the number of fields defined for the // proto. #if defined(PROTOBUF_CUSTOM_VTABLE) - size_t ByteSizeLong() const; + size_t ByteSizeLong() const { return _class_data_->byte_size_long(*this); } #else virtual size_t ByteSizeLong() const = 0; #endif // PROTOBUF_CUSTOM_VTABLE @@ -512,7 +514,11 @@ class PROTOBUF_EXPORT MessageLite { // sub-message is changed, all of its parents' cached sizes would need to be // invalidated, which is too much work for an otherwise inlined setter // method.) +#if defined(PROTOBUF_CUSTOM_VTABLE) + int GetCachedSize() const { return AccessCachedSize().Get(); } +#else int GetCachedSize() const; +#endif const char* _InternalParse(const char* ptr, internal::ParseContext* ctx); @@ -795,7 +801,9 @@ class PROTOBUF_EXPORT MessageLite { // uint8_t* _InternalSerialize(uint8_t* ptr) const; #if defined(PROTOBUF_CUSTOM_VTABLE) uint8_t* _InternalSerialize(uint8_t* ptr, - io::EpsCopyOutputStream* stream) const; + io::EpsCopyOutputStream* stream) const { + return _class_data_->serialize(*this, ptr, stream); + } #else // PROTOBUF_CUSTOM_VTABLE virtual uint8_t* _InternalSerialize( uint8_t* ptr, io::EpsCopyOutputStream* stream) const = 0; From 65cb2c406a554bd7ed81cdaeb01c812b9b8b3071 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 19 Jul 2024 11:45:58 -0700 Subject: [PATCH 034/107] Automated rollback of commit 8d7f50f7c205a81421a08debd4bb16eabd3a04e5. PiperOrigin-RevId: 654076634 --- src/google/protobuf/descriptor.cc | 41 ++++++++++--- src/google/protobuf/descriptor.h | 19 +++--- src/google/protobuf/descriptor_lite.h | 11 ++++ src/google/protobuf/descriptor_unittest.cc | 70 ++++++++++++++++++++++ 4 files changed, 125 insertions(+), 16 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index ed5f6af920..62a2dd58cc 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3939,6 +3939,29 @@ bool FieldDescriptor::has_optional_keyword() const { is_optional() && !containing_oneof()); } +FieldDescriptor::CppStringType FieldDescriptor::cpp_string_type() const { + ABSL_DCHECK(cpp_type() == FieldDescriptor::CPPTYPE_STRING); + switch (features().GetExtension(pb::cpp).string_type()) { + case pb::CppFeatures::VIEW: + return CppStringType::kView; + case pb::CppFeatures::CORD: + // In open-source, protobuf CORD is only supported for singular bytes + // fields. + if (type() != FieldDescriptor::TYPE_BYTES || is_repeated() || + is_extension()) { + return CppStringType::kString; + } + return CppStringType::kCord; + case pb::CppFeatures::STRING: + return CppStringType::kString; + default: + // If features haven't been resolved, this is a dynamic build not for C++ + // codegen. Just use string type. + ABSL_DCHECK(!features().GetExtension(pb::cpp).has_string_type()); + return CppStringType::kString; + } +} + // Location methods =============================================== bool FileDescriptor::GetSourceLocation(const std::vector& path, @@ -5476,8 +5499,7 @@ static void EnforceCTypeStringTypeConsistency( Edition edition, FieldDescriptor::CppType type, const pb::CppFeatures& cpp_features, FieldOptions& options) { if (&options == &FieldOptions::default_instance()) return; - if (edition < Edition::EDITION_2024 && - type == FieldDescriptor::CPPTYPE_STRING) { + if (type == FieldDescriptor::CPPTYPE_STRING) { switch (cpp_features.string_type()) { case pb::CppFeatures::CORD: options.set_ctype(FieldOptions::CORD); @@ -6150,6 +6172,14 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( }); internal::VisitDescriptors(*result, [&](const FieldDescriptor& field) { + if (result->edition() >= Edition::EDITION_2024 && + field.options().has_ctype()) { + // "ctype" is no longer supported in edition 2024 and beyond. + AddError( + field.full_name(), proto, DescriptorPool::ErrorCollector::NAME, + "ctype option is not allowed under edition 2024 and beyond. Use " + "the feature string_type = VIEW|CORD|STRING|... instead."); + } EnforceCTypeStringTypeConsistency( field.file()->edition(), field.cpp_type(), field.merged_features_->GetExtension(pb::cpp), @@ -7904,12 +7934,7 @@ void DescriptorBuilder::ValidateOptions(const FieldDescriptor* field, auto& field_options = field->options(); if (field_options.has_ctype()) { - if (edition >= Edition::EDITION_2024) { - // "ctype" is no longer supported in edition 2024 and beyond. - AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, - "ctype option is not allowed under edition 2024 and beyond. Use " - "the feature string_type = VIEW|CORD|STRING|... instead."); - } else if (edition == Edition::EDITION_2023) { + if (edition == Edition::EDITION_2023) { if (field->cpp_type() != FieldDescriptor::CPPTYPE_STRING) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 28db34d95e..08a3d5c510 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -875,6 +875,10 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase, const char* cpp_type_name() const; // Name of the C++ type. Label label() const; // optional/required/repeated +#ifndef SWIG + CppStringType cpp_string_type() const; // The C++ string type of this field. +#endif + bool is_required() const; // shorthand for label() == LABEL_REQUIRED bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL bool is_repeated() const; // shorthand for label() == LABEL_REPEATED @@ -2932,22 +2936,21 @@ PROTOBUF_EXPORT bool HasPreservingUnknownEnumSemantics( PROTOBUF_EXPORT bool HasHasbit(const FieldDescriptor* field); +#ifndef SWIG // For a string field, returns the effective ctype. If the actual ctype is // not supported, returns the default of STRING. template typename FieldOpts::CType EffectiveStringCType(const FieldDesc* field) { - ABSL_DCHECK(field->cpp_type() == FieldDescriptor::CPPTYPE_STRING); - // Open-source protobuf release only supports STRING ctype and CORD for - // sinuglar bytes. - if (field->type() == FieldDescriptor::TYPE_BYTES && !field->is_repeated() && - field->options().ctype() == FieldOpts::CORD && !field->is_extension()) { - return FieldOpts::CORD; + // TODO Replace this function with FieldDescriptor::string_type; + switch (field->cpp_string_type()) { + case FieldDescriptor::CppStringType::kCord: + return FieldOpts::CORD; + default: + return FieldOpts::STRING; } - return FieldOpts::STRING; } -#ifndef SWIG enum class Utf8CheckMode : uint8_t { kStrict = 0, // Parsing will fail if non UTF-8 data is in string fields. kVerify = 1, // Only log an error but parsing will succeed. diff --git a/src/google/protobuf/descriptor_lite.h b/src/google/protobuf/descriptor_lite.h index db5805affe..e1a90b7fa6 100644 --- a/src/google/protobuf/descriptor_lite.h +++ b/src/google/protobuf/descriptor_lite.h @@ -78,6 +78,17 @@ class FieldDescriptorLite { MAX_LABEL = 3, // Constant useful for defining lookup tables // indexed by Label. }; + + // Identifies the storage type of a C++ string field. This corresponds to + // pb.CppFeatures.StringType, but is compatible with ctype prior to Edition + // 2024. 0 is reserved for errors. +#ifndef SWIG + enum class CppStringType { + kView = 1, + kCord = 2, + kString = 3, + }; +#endif }; } // namespace internal diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 0c13b3a2b4..f4d3137099 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -7594,6 +7594,9 @@ TEST_F(FeaturesTest, Proto2Features) { EXPECT_TRUE(message->FindFieldByName("req")->is_required()); EXPECT_TRUE(file->enum_type(0)->is_closed()); + EXPECT_EQ(message->FindFieldByName("str")->cpp_string_type(), + FieldDescriptor::CppStringType::kString); + // Check round-trip consistency. FileDescriptorProto proto; file->CopyTo(&proto); @@ -9710,6 +9713,73 @@ TEST_F(FeaturesTest, EnumFeatureHelpers) { EXPECT_FALSE(HasPreservingUnknownEnumSemantics(field_legacy_closed)); } +TEST_F(FeaturesTest, FieldCppStringType) { + BuildDescriptorMessagesInTestPool(); + const std::string file_contents = absl::Substitute( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2024 + message_type { + name: "Foo" + field { + name: "view" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + } + field { + name: "str" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_STRING + options { + features { + [pb.cpp] { string_type: STRING } + } + } + } + field { + name: "cord" + number: 3 + label: LABEL_OPTIONAL + type: TYPE_STRING + options { + features { + [pb.cpp] { string_type: CORD } + } + } + } + field { + name: "cord_bytes" + number: 4 + label: LABEL_OPTIONAL + type: TYPE_BYTES + options { + features { + [pb.cpp] { string_type: CORD } + } + } + } $0 + } + )pb", + "" + ); + const FileDescriptor* file = BuildFile(file_contents); + const Descriptor* message = file->message_type(0); + const FieldDescriptor* view = message->field(0); + const FieldDescriptor* str = message->field(1); + const FieldDescriptor* cord = message->field(2); + const FieldDescriptor* cord_bytes = message->field(3); + + EXPECT_EQ(view->cpp_string_type(), FieldDescriptor::CppStringType::kView); + EXPECT_EQ(str->cpp_string_type(), FieldDescriptor::CppStringType::kString); + EXPECT_EQ(cord_bytes->cpp_string_type(), + FieldDescriptor::CppStringType::kCord); + EXPECT_EQ(cord->cpp_string_type(), FieldDescriptor::CppStringType::kString); + +} + TEST_F(FeaturesTest, MergeFeatureValidationFailed) { BuildDescriptorMessagesInTestPool(); BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); From b698355957a31aedf35bb4b954ffbf2f2c7edb72 Mon Sep 17 00:00:00 2001 From: Tony Liao Date: Fri, 19 Jul 2024 13:48:39 -0700 Subject: [PATCH 035/107] No longer emit an open brace in MayEmitIfNonDefaultCheck. Not always emitting the opening brace in this function call can give the caller a bit more flexibility: for example, it allows for easier use of formatted string substitutions when both the opening and the closing brace can be accounted for by the same caller. PiperOrigin-RevId: 654113509 --- src/google/protobuf/compiler/cpp/message.cc | 38 +++++++++++++-------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 4756acf4bc..2bd41be347 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -227,10 +227,16 @@ bool MayEmitIfNonDefaultCheck(io::Printer* p, const std::string& prefix, ABSL_CHECK(!HasHasbit(field)); if (!ShouldEmitNonDefaultCheck(field)) return false; + // SUBTLE: format_string must be a raw string without newline. + // io::Printer::Emit treats the format string as a "raw string" if it doesn't + // contain multiple lines. Note that a string of the form "\n($condition$)\n" + // (i.e. newline characters are present; there is only one non-empty line) + // will be treated as a multi-line string. + // + // io::Printer::Emit will print a newline if the input is a multi-line string. + absl::string_view format_string = "if ($condition$) "; p->Emit({{"condition", [&] { EmitNonDefaultCheck(p, prefix, field); }}}, - R"cc( - if ($condition$) { - )cc"); + format_string); return true; } @@ -3952,12 +3958,15 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) { } else if (field->is_optional() && !HasHasbit(field)) { // Merge semantics without true field presence: primitive fields are // merged only if non-zero (numeric) or non-empty (string). - bool have_enclosing_if = MayEmitIfNonDefaultCheck(p, "from.", field); - if (have_enclosing_if) format.Indent(); + bool emitted_check = MayEmitIfNonDefaultCheck(p, "from.", field); + if (emitted_check) { + p->Emit("{\n"); + p->Indent(); + } generator.GenerateMergingCode(p); - if (have_enclosing_if) { - format.Outdent(); - format("}\n"); + if (emitted_check) { + p->Outdent(); + p->Emit("}\n"); } } else if (field->options().weak() || cached_has_word_index != HasWordIndex(field)) { @@ -4225,14 +4234,15 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* p, } )cc"); } else if (field->is_optional()) { - bool have_enclosing_if = MayEmitIfNonDefaultCheck(p, "this_.", field); - if (have_enclosing_if) p->Indent(); + bool emitted_check = MayEmitIfNonDefaultCheck(p, "this_.", field); + if (emitted_check) { + p->Emit("{\n"); + p->Indent(); + } emit_body(); - if (have_enclosing_if) { + if (emitted_check) { p->Outdent(); - p->Emit(R"cc( - } - )cc"); + p->Emit("}\n"); } } else { emit_body(); From 74f6da4d26173088de178365678891105393799e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Jul 2024 14:42:20 -0700 Subject: [PATCH 036/107] Switch C++ extension identifier to constexpr. Add static_assert test for ExtensionNumber API. PiperOrigin-RevId: 654129635 --- hpb/hpb.h | 9 +++-- hpb_generator/gen_extensions.cc | 50 ++++++++------------------ hpb_generator/gen_extensions.h | 8 +---- hpb_generator/gen_messages.cc | 21 +---------- hpb_generator/protoc-gen-upb-protos.cc | 5 ++- hpb_generator/tests/test_generated.cc | 2 ++ 6 files changed, 24 insertions(+), 71 deletions(-) diff --git a/hpb/hpb.h b/hpb/hpb.h index 7ae1b605e8..9e18057638 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -204,13 +204,12 @@ class ExtensionIdentifier : public ExtensionMiniTableProvider { using Extendee = ExtendeeType; constexpr explicit ExtensionIdentifier( - const upb_MiniTableExtension* mini_table_ext) - : ExtensionMiniTableProvider(mini_table_ext) {} + uint32_t number, const upb_MiniTableExtension* mini_table_ext) + : ExtensionMiniTableProvider(mini_table_ext), number_(number) {} private: - constexpr uint32_t number() const { - return upb_MiniTableExtension_Number(mini_table_ext()); - } + uint32_t number_; + constexpr uint32_t number() const { return number_; } friend class PrivateAccess; }; diff --git a/hpb_generator/gen_extensions.cc b/hpb_generator/gen_extensions.cc index d0c54eb36d..1bfe74b555 100644 --- a/hpb_generator/gen_extensions.cc +++ b/hpb_generator/gen_extensions.cc @@ -7,9 +7,14 @@ #include "google/protobuf/compiler/hpb/gen_extensions.h" +#include +#include +#include + #include "absl/strings/str_cat.h" -#include "google/protobuf/compiler/hpb/gen_utils.h" #include "google/protobuf/compiler/hpb/names.h" +#include "google/protobuf/compiler/hpb/output.h" +#include "google/protobuf/descriptor.h" namespace google::protobuf::hpb_generator { @@ -38,55 +43,28 @@ void WriteExtensionIdentifierHeader(const protobuf::FieldDescriptor* ext, if (ext->extension_scope()) { output( R"cc( - static const ::protos::internal::ExtensionIdentifier<$0, $1> $2; - )cc", - ContainingTypeName(ext), CppTypeParameterName(ext), ext->name()); - } else { - output( - R"cc( - extern const ::protos::internal::ExtensionIdentifier<$0, $1> $2; - )cc", - ContainingTypeName(ext), CppTypeParameterName(ext), ext->name()); - } -} - -void WriteExtensionIdentifiersHeader( - const std::vector& extensions, - Output& output) { - for (const auto* ext : extensions) { - if (!ext->extension_scope()) { - WriteExtensionIdentifierHeader(ext, output); - } - } -} - -void WriteExtensionIdentifier(const protobuf::FieldDescriptor* ext, - Output& output) { - std::string mini_table_name = - absl::StrCat(ExtensionIdentifierBase(ext), "_", ext->name(), "_ext"); - if (ext->extension_scope()) { - output( - R"cc( - const ::protos::internal::ExtensionIdentifier<$0, $3> $4::$2(&$1); + static constexpr ::protos::internal::ExtensionIdentifier<$0, $3> $2{ + $4, &$1}; )cc", ContainingTypeName(ext), mini_table_name, ext->name(), - CppTypeParameterName(ext), ClassName(ext->extension_scope())); + CppTypeParameterName(ext), ext->number()); } else { output( R"cc( - const ::protos::internal::ExtensionIdentifier<$0, $3> $2(&$1); + inline constexpr ::protos::internal::ExtensionIdentifier<$0, $3> $2{ + $4, &$1}; )cc", ContainingTypeName(ext), mini_table_name, ext->name(), - CppTypeParameterName(ext)); + CppTypeParameterName(ext), ext->number()); } } -void WriteExtensionIdentifiers( +void WriteExtensionIdentifiersHeader( const std::vector& extensions, Output& output) { for (const auto* ext : extensions) { if (!ext->extension_scope()) { - WriteExtensionIdentifier(ext, output); + WriteExtensionIdentifierHeader(ext, output); } } } diff --git a/hpb_generator/gen_extensions.h b/hpb_generator/gen_extensions.h index e48f657e8d..1d2e932c43 100644 --- a/hpb_generator/gen_extensions.h +++ b/hpb_generator/gen_extensions.h @@ -8,8 +8,8 @@ #ifndef PROTOBUF_COMPILER_HBP_GEN_EXTENSIONS_H_ #define PROTOBUF_COMPILER_HBP_GEN_EXTENSIONS_H_ -#include "google/protobuf/descriptor.h" #include "google/protobuf/compiler/hpb/output.h" +#include "google/protobuf/descriptor.h" namespace google::protobuf::hpb_generator { @@ -20,12 +20,6 @@ void WriteExtensionIdentifiersHeader( Output& output); void WriteExtensionIdentifierHeader(const protobuf::FieldDescriptor* ext, Output& output); -void WriteExtensionIdentifiers( - const std::vector& extensions, - Output& output); -void WriteExtensionIdentifier(const protobuf::FieldDescriptor* ext, - Output& output); - } // namespace protobuf } // namespace google::hpb_generator diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index 75e486ed24..d2576743c4 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -15,15 +15,14 @@ #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "google/protobuf/descriptor.h" #include "google/protobuf/compiler/hpb/gen_accessors.h" #include "google/protobuf/compiler/hpb/gen_enums.h" #include "google/protobuf/compiler/hpb/gen_extensions.h" #include "google/protobuf/compiler/hpb/gen_utils.h" #include "google/protobuf/compiler/hpb/names.h" #include "google/protobuf/compiler/hpb/output.h" +#include "google/protobuf/descriptor.h" #include "upb_generator/common.h" -#include "upb_generator/file_layout.h" namespace google::protobuf::hpb_generator { @@ -48,10 +47,6 @@ void WriteInternalForwardDeclarationsInHeader( const protobuf::Descriptor* message, Output& output); void WriteDefaultInstanceHeader(const protobuf::Descriptor* message, Output& output); -void WriteExtensionIdentifiersImplementation( - const protobuf::Descriptor* message, - const std::vector& file_exts, - Output& output); void WriteUsingEnumsInHeader( const protobuf::Descriptor* message, const std::vector& file_enums, @@ -457,8 +452,6 @@ void WriteMessageImplementation( } )cc", ClassName(descriptor)); - - WriteExtensionIdentifiersImplementation(descriptor, file_exts, output); } } @@ -486,18 +479,6 @@ void WriteExtensionIdentifiersInClassHeader( } } -void WriteExtensionIdentifiersImplementation( - const protobuf::Descriptor* message, - const std::vector& file_exts, - Output& output) { - for (auto* ext : file_exts) { - if (ext->extension_scope() && - ext->extension_scope()->full_name() == message->full_name()) { - WriteExtensionIdentifier(ext, output); - } - } -} - void WriteUsingEnumsInHeader( const protobuf::Descriptor* message, const std::vector& file_enums, diff --git a/hpb_generator/protoc-gen-upb-protos.cc b/hpb_generator/protoc-gen-upb-protos.cc index d946fb260a..5bf9b8f381 100644 --- a/hpb_generator/protoc-gen-upb-protos.cc +++ b/hpb_generator/protoc-gen-upb-protos.cc @@ -13,14 +13,14 @@ #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/compiler/code_generator.h" -#include "google/protobuf/compiler/plugin.h" -#include "google/protobuf/descriptor.h" #include "google/protobuf/compiler/hpb/gen_enums.h" #include "google/protobuf/compiler/hpb/gen_extensions.h" #include "google/protobuf/compiler/hpb/gen_messages.h" #include "google/protobuf/compiler/hpb/gen_utils.h" #include "google/protobuf/compiler/hpb/names.h" #include "google/protobuf/compiler/hpb/output.h" +#include "google/protobuf/compiler/plugin.h" +#include "google/protobuf/descriptor.h" namespace google::protobuf::hpb_generator { namespace { @@ -211,7 +211,6 @@ void WriteSource(const protobuf::FileDescriptor* file, Output& output, WriteMessageImplementations(file, output); const std::vector this_file_exts = SortedExtensions(file); - WriteExtensionIdentifiers(this_file_exts, output); WriteEndNamespace(file, output); output("#include \"upb/port/undef.inc\"\n\n"); diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index c007d31d7c..578f2d8b8d 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -1242,6 +1242,8 @@ TEST(CppGeneratedCode, FieldNumberConstants) { } TEST(CppGeneratedCode, ExtensionFieldNumberConstant) { + static_assert(::protos::ExtensionNumber(ThemeExtension::theme_extension) == + 12003); EXPECT_EQ(12003, ::protos::ExtensionNumber(ThemeExtension::theme_extension)); } From 630dbfc964e1c09bb245968e7ccc1871412f36f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Sat, 20 Jul 2024 08:45:58 -0700 Subject: [PATCH 037/107] Internal change. PiperOrigin-RevId: 654315587 --- src/google/protobuf/compiler/java/lite/message.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index 6667e7aed7..2e1ecd38a0 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -260,6 +260,10 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) { "}\n" "\n"); } + if (!context_->options().opensource_runtime) { + printer->Print( + "@com.google.protobuf.Internal.ProtoMethodMayReturnNull\n"); + } printer->Print( vars, "public static $oneof_capitalized_name$Case forNumber(int value) {\n" From aa47311a832da6fcf32b998378622ae3bdaa4c21 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 05:14:17 -0700 Subject: [PATCH 038/107] Use toolchain helpers in py_proto_library The change is a no-op. Just reduces code duplication. PiperOrigin-RevId: 654702701 --- bazel/BUILD.bazel | 1 + bazel/py_proto_library.bzl | 12 +++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 1d6749dd76..d0546f9b30 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -40,6 +40,7 @@ bzl_library( deps = [ "//bazel/common:proto_common_bzl", "//bazel/common:proto_info_bzl", + "//bazel/private:toolchain_helpers_bzl", "@rules_python//python:py_info_bzl", ], ) diff --git a/bazel/py_proto_library.bzl b/bazel/py_proto_library.bzl index 86af999502..6e3d041f36 100644 --- a/bazel/py_proto_library.bzl +++ b/bazel/py_proto_library.bzl @@ -3,6 +3,7 @@ load("@rules_python//python:py_info.bzl", "PyInfo") load("//bazel/common:proto_common.bzl", "proto_common") load("//bazel/common:proto_info.bzl", "ProtoInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" @@ -22,9 +23,6 @@ _PyProtoInfo = provider( def _filter_provider(provider, *attrs): return [dep[provider] for attr in attrs for dep in attr if provider in dep] -def _incompatible_toolchains_enabled(): - return getattr(proto_common, "INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION", False) - def _py_proto_aspect_impl(target, ctx): """Generates and compiles Python code for a proto_library. @@ -51,7 +49,7 @@ def _py_proto_aspect_impl(target, ctx): proto.path, )) - if _incompatible_toolchains_enabled(): + if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: toolchain = ctx.toolchains[PY_PROTO_TOOLCHAIN] if not toolchain: fail("No toolchains registered for '%s'." % PY_PROTO_TOOLCHAIN) @@ -120,15 +118,15 @@ def _py_proto_aspect_impl(target, ctx): _py_proto_aspect = aspect( implementation = _py_proto_aspect_impl, - attrs = {} if _incompatible_toolchains_enabled() else { + attrs = toolchains.if_legacy_toolchain({ "_aspect_proto_toolchain": attr.label( default = "//python:python_toolchain", ), - }, + }), attr_aspects = ["deps"], required_providers = [ProtoInfo], provides = [_PyProtoInfo], - toolchains = [PY_PROTO_TOOLCHAIN] if _incompatible_toolchains_enabled() else [], + toolchains = toolchains.use_toolchain(PY_PROTO_TOOLCHAIN), ) def _py_proto_library_rule(ctx): From 0a917b95f27bab71caa49f0ebe88931189260044 Mon Sep 17 00:00:00 2001 From: Dmitri Gribenko Date: Mon, 22 Jul 2024 06:50:36 -0700 Subject: [PATCH 039/107] Use `#[googletest::test]` in Protobuf Rust PiperOrigin-RevId: 654726974 --- rust/cpp.rs | 4 +- rust/map.rs | 16 ++-- rust/proxied.rs | 14 +-- rust/repeated.rs | 6 +- rust/string.rs | 12 +-- rust/test/cpp/debug_test.rs | 6 +- rust/test/cpp/interop/main.rs | 14 +-- rust/test/shared/BUILD | 14 ++- rust/test/shared/accessors_map_test.rs | 14 +-- rust/test/shared/accessors_proto3_test.rs | 28 +++--- rust/test/shared/accessors_repeated_test.rs | 22 ++--- rust/test/shared/accessors_test.rs | 94 +++++++++---------- rust/test/shared/bad_names_test.rs | 6 +- rust/test/shared/child_parent_test.rs | 6 +- rust/test/shared/edition2023_test.rs | 6 +- rust/test/shared/enum_test.rs | 28 +++--- .../shared/fields_with_imported_types_test.rs | 6 +- rust/test/shared/import_public_test.rs | 2 +- rust/test/shared/merge_from_test.rs | 10 +- rust/test/shared/nested_types_test.rs | 4 +- rust/test/shared/package_test.rs | 4 +- rust/test/shared/proto_macro_test.rs | 14 +-- rust/test/shared/serialization_test.rs | 14 +-- rust/test/shared/simple_nested_test.rs | 14 +-- rust/test/shared/utf8/utf8_test.rs | 6 +- rust/test/upb/debug_string_test.rs | 2 +- rust/test/upb/string_ctypes_test.rs | 6 +- 27 files changed, 191 insertions(+), 181 deletions(-) diff --git a/rust/cpp.rs b/rust/cpp.rs index d8e4df041a..f852dfdfbc 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -899,14 +899,14 @@ mod tests { (content.as_mut_ptr(), content.len()) } - #[test] + #[googletest::test] fn test_serialized_data_roundtrip() { let (ptr, len) = allocate_byte_array(b"Hello world"); let serialized_data = SerializedData { data: NonNull::new(ptr).unwrap(), len }; assert_that!(&*serialized_data, eq(b"Hello world")); } - #[test] + #[googletest::test] fn test_empty_string() { let empty_str: String = RustStringRawParts { data: std::ptr::null(), len: 0 }.into(); assert_that!(empty_str, eq("")); diff --git a/rust/map.rs b/rust/map.rs index a0e0c9e449..170f50c97d 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -464,7 +464,7 @@ mod tests { use crate::{ProtoBytes, ProtoStr, ProtoString}; use googletest::prelude::*; - #[test] + #[googletest::test] fn test_proxied_scalar() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -490,7 +490,7 @@ mod tests { assert_that!(map_view_4.is_empty(), eq(false)); } - #[test] + #[googletest::test] fn test_proxied_str() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -515,7 +515,7 @@ mod tests { assert_that!(map_view_4.is_empty(), eq(false)); } - #[test] + #[googletest::test] fn test_proxied_iter() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -562,7 +562,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_overwrite_insert() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -572,7 +572,7 @@ mod tests { assert_that!(map.as_mut(), unordered_elements_are![eq((0, ProtoStr::from_str("buzz"))),]); } - #[test] + #[googletest::test] fn test_extend() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -609,7 +609,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_copy_from() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -640,7 +640,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_all_maps_can_be_constructed() { macro_rules! gen_proto_values { ($key_t:ty, $($value_t:ty),*) => { @@ -662,7 +662,7 @@ mod tests { gen_proto_keys!(i32, u32, i64, u64, bool, ProtoString); } - #[test] + #[googletest::test] fn test_dbg() { let mut map = Map::::new(); assert_that!(format!("{:?}", map.as_view()), eq("MapView(\"i32\", \"f64\")")); diff --git a/rust/proxied.rs b/rust/proxied.rs index e90dbe5805..5606824441 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -312,7 +312,7 @@ mod tests { } } - #[test] + #[googletest::test] fn test_as_view() { let my_proxied = MyProxied { val: "Hello World".to_string() }; @@ -327,7 +327,7 @@ mod tests { x.into_view() // OK: we return the same lifetime as we got in. } - #[test] + #[googletest::test] fn test_mut_into_view() { let mut my_proxied = MyProxied { val: "Hello World".to_string() }; reborrow_mut_into_view(my_proxied.as_mut()); @@ -335,7 +335,7 @@ mod tests { fn require_unified_lifetimes<'msg>(_x: Mut<'msg, MyProxied>, _y: View<'msg, MyProxied>) {} - #[test] + #[googletest::test] fn test_require_unified_lifetimes() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let my_mut = my_proxied.as_mut(); @@ -360,7 +360,7 @@ mod tests { [x.as_view(), y.as_view()] } - #[test] + #[googletest::test] fn test_reborrow_generic_as_view() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let mut my_mut = my_proxied.as_mut(); @@ -387,7 +387,7 @@ mod tests { [x.into_view(), y] } - #[test] + #[googletest::test] fn test_reborrow_generic_into_view() { let my_proxied = MyProxied { val: "Hello1".to_string() }; let my_view = my_proxied.as_view(); @@ -407,7 +407,7 @@ mod tests { [x.into_view(), y] } - #[test] + #[googletest::test] fn test_reborrow_generic_mut_into_view() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let my_mut = my_proxied.as_mut(); @@ -430,7 +430,7 @@ mod tests { [x.into_mut(), y] } - #[test] + #[googletest::test] fn test_reborrow_generic_mut_into_mut() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let my_mut = my_proxied.as_mut(); diff --git a/rust/repeated.rs b/rust/repeated.rs index 692a24971f..d7a02e4c5d 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -542,7 +542,7 @@ mod tests { use super::*; use googletest::prelude::*; - #[test] + #[googletest::test] fn test_primitive_repeated() { macro_rules! primitive_repeated_tests { ($($t:ty => [$($vals:expr),* $(,)?]),* $(,)?) => { @@ -583,7 +583,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_repeated_extend() { let mut r = Repeated::::new(); @@ -600,7 +600,7 @@ mod tests { assert_that!(r.as_mut(), elements_are![eq(0), eq(1), eq(2), eq(3)]); } - #[test] + #[googletest::test] fn test_repeated_iter_into_proxied() { let r: Repeated = [0, 1, 2, 3].into_iter().into_proxied(Private); assert_that!(r.as_view(), elements_are![eq(0), eq(1), eq(2), eq(3)]); diff --git a/rust/string.rs b/rust/string.rs index 6ebdd37af1..e3b8365b40 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -545,7 +545,7 @@ mod tests { // UTF-8 test cases copied from: // https://github.com/rust-lang/rust/blob/e8ee0b7/library/core/tests/str_lossy.rs - #[test] + #[googletest::test] fn proto_str_debug() { assert_eq!(&format!("{:?}", test_proto_str(b"Hello There")), "\"Hello There\""); assert_eq!( @@ -557,7 +557,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn proto_str_display() { assert_eq!(&test_proto_str(b"Hello There").to_string(), "Hello There"); assert_eq!( @@ -566,7 +566,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn proto_str_to_rust_str() { assert_eq!(test_proto_str(b"hello").to_str(), Ok("hello")); assert_eq!(test_proto_str("ศไทย中华Việt Nam".as_bytes()).to_str(), Ok("ศไทย中华Việt Nam")); @@ -583,7 +583,7 @@ mod tests { } } - #[test] + #[googletest::test] fn proto_str_to_cow() { assert_eq!(test_proto_str(b"hello").to_cow_lossy(), Cow::Borrowed("hello")); assert_eq!( @@ -605,7 +605,7 @@ mod tests { } } - #[test] + #[googletest::test] fn proto_str_utf8_chunks() { macro_rules! assert_chunks { ($bytes:expr, $($chunks:expr),* $(,)?) => { @@ -684,7 +684,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn proto_str_chars() { macro_rules! assert_chars { ($bytes:expr, $chars:expr) => { diff --git a/rust/test/cpp/debug_test.rs b/rust/test/cpp/debug_test.rs index 82b2d1b773..2b784a9a9b 100644 --- a/rust/test/cpp/debug_test.rs +++ b/rust/test/cpp/debug_test.rs @@ -3,7 +3,7 @@ use googletest::prelude::*; use optimize_for_lite_rust_proto::OptimizeForLiteTestMessage; #[cfg(not(lite_runtime))] -#[test] +#[googletest::test] fn test_debug() { let mut msg = DebugMsg::new(); msg.set_id(1); @@ -14,7 +14,7 @@ fn test_debug() { } #[cfg(lite_runtime)] -#[test] +#[googletest::test] fn test_debug_lite() { let mut msg = DebugMsg::new(); msg.set_id(1); @@ -27,7 +27,7 @@ fn test_debug_lite() { /// A message with the option set to optimize for lite will behave as a lite /// message regardless of the `lite_runtime` feature. Run this test not guarded /// by the cfg(lite_runtime) and ensure it functions as lite. -#[test] +#[googletest::test] fn test_optimize_for_lite_option() { let mut msg = OptimizeForLiteTestMessage::new(); msg.set_value("password"); diff --git a/rust/test/cpp/interop/main.rs b/rust/test/cpp/interop/main.rs index 812dab3a59..93ec9e9f0e 100644 --- a/rust/test/cpp/interop/main.rs +++ b/rust/test/cpp/interop/main.rs @@ -33,7 +33,7 @@ extern "C" { fn GetBytesExtension(msg: RawMessage) -> PtrAndLen; } -#[test] +#[googletest::test] fn send_to_cpp() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int32(7); @@ -43,7 +43,7 @@ fn send_to_cpp() { assert_eq!(i, 7); } -#[test] +#[googletest::test] fn mutate_message_mut_in_cpp() { let mut msg1 = TestAllTypes::new(); unsafe { @@ -58,7 +58,7 @@ fn mutate_message_mut_in_cpp() { proto_assert_eq!(msg1, msg2); } -#[test] +#[googletest::test] fn deserialize_in_rust() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -71,7 +71,7 @@ fn deserialize_in_rust() { proto_assert_eq!(msg1, msg2); } -#[test] +#[googletest::test] fn deserialize_in_cpp() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -88,7 +88,7 @@ fn deserialize_in_cpp() { proto_assert_eq!(msg1, msg2); } -#[test] +#[googletest::test] fn deserialize_in_cpp_into_mut() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -106,7 +106,7 @@ fn deserialize_in_cpp_into_mut() { } } -#[test] +#[googletest::test] fn deserialize_in_cpp_into_view() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -126,7 +126,7 @@ fn deserialize_in_cpp_into_view() { // This test ensures that random fields we (Rust) don't know about don't // accidentally get destroyed by Rust. -#[test] +#[googletest::test] fn smuggle_extension() { let msg1 = unsafe { TestAllExtensions::__unstable_wrap_cpp_grant_permission_to_break(NewWithExtension()) diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index af009e9c08..e39ee34976 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -87,6 +87,7 @@ rust_test( srcs = ["import_public_test.rs"], deps = [ "//rust/test:import_public_cpp_rust_proto", + "@crate_index//:googletest", ], ) @@ -95,6 +96,7 @@ rust_test( srcs = ["import_public_test.rs"], deps = [ "//rust/test:import_public_upb_rust_proto", + "@crate_index//:googletest", ], ) @@ -105,6 +107,7 @@ rust_test( "//rust/test:dots_in_package_cpp_rust_proto", "//rust/test:no_package_cpp_rust_proto", "//rust/test:package_cpp_rust_proto", + "@crate_index//:googletest", ], ) @@ -115,6 +118,7 @@ rust_test( "//rust/test:dots_in_package_upb_rust_proto", "//rust/test:no_package_upb_rust_proto", "//rust/test:package_upb_rust_proto", + "@crate_index//:googletest", ], ) @@ -155,13 +159,19 @@ rust_test( rust_test( name = "nested_types_cpp_test", srcs = ["nested_types_test.rs"], - deps = ["//rust/test:unittest_cpp_rust_proto"], + deps = [ + "//rust/test:unittest_cpp_rust_proto", + "@crate_index//:googletest", + ], ) rust_test( name = "nested_types_upb_test", srcs = ["nested_types_test.rs"], - deps = ["//rust/test:unittest_upb_rust_proto"], + deps = [ + "//rust/test:unittest_upb_rust_proto", + "@crate_index//:googletest", + ], ) rust_test( diff --git a/rust/test/shared/accessors_map_test.rs b/rust/test/shared/accessors_map_test.rs index 26a309de28..b284a24752 100644 --- a/rust/test/shared/accessors_map_test.rs +++ b/rust/test/shared/accessors_map_test.rs @@ -19,7 +19,7 @@ macro_rules! generate_map_primitives_tests { $(,)? ) => { paste! { $( - #[test] + #[googletest::test] fn [< test_map_ $k_field _ $v_field >]() { let mut msg = TestMap::new(); assert_that!(msg.[< map_ $k_field _ $v_field >]().len(), eq(0)); @@ -95,7 +95,7 @@ generate_map_primitives_tests!( (i32, MapEnum, int32, enum, 1, MapEnum::Baz), ); -#[test] +#[googletest::test] fn collect_as_hashmap() { // Highlights conversion from protobuf map to hashmap. let mut msg = TestMap::new(); @@ -114,7 +114,7 @@ fn collect_as_hashmap() { ); } -#[test] +#[googletest::test] fn test_string_maps() { let mut msg = TestMap::new(); msg.map_string_string_mut().insert("hello", "world"); @@ -126,7 +126,7 @@ fn test_string_maps() { assert_that!(msg.map_string_string().len(), eq(0)); } -#[test] +#[googletest::test] fn test_nested_enum_maps() { // Verify that C++ thunks are generated and are with the right name for strings TestMapWithNestedEnum::new() @@ -134,7 +134,7 @@ fn test_nested_enum_maps() { .insert("foo", test_map_with_nested_enum::inner_nested::NestedEnum::Foo); } -#[test] +#[googletest::test] fn test_bytes_and_string_copied() { let mut msg = TestMap::new(); @@ -154,7 +154,7 @@ fn test_bytes_and_string_copied() { assert_that!(msg.map_int32_bytes_mut().get(1).unwrap(), eq(b"world")); } -#[test] +#[googletest::test] fn test_map_setter() { let mut msg = TestMap::new(); msg.map_string_string_mut().insert("hello", "world"); @@ -177,7 +177,7 @@ macro_rules! generate_map_with_msg_values_tests { $(,)? ) => { paste! { $( - #[test] + #[googletest::test] fn [< test_map_ $k_field _all_types >]() { // We need to cover the following upb/c++ thunks: // TODO - b/323883851: Add test once Map::new is public. diff --git a/rust/test/shared/accessors_proto3_test.rs b/rust/test/shared/accessors_proto3_test.rs index f4096ea43a..63aaf1acf5 100644 --- a/rust/test/shared/accessors_proto3_test.rs +++ b/rust/test/shared/accessors_proto3_test.rs @@ -12,7 +12,7 @@ use protobuf::Optional; use unittest_proto3_optional_rust_proto::{test_proto3_optional, TestProto3Optional}; use unittest_proto3_rust_proto::{test_all_types, TestAllTypes}; -#[test] +#[googletest::test] fn test_fixed32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_fixed32(), eq(0)); @@ -27,7 +27,7 @@ fn test_fixed32_accessors() { assert_that!(msg.optional_fixed32(), eq(43)); } -#[test] +#[googletest::test] fn test_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_bool(), eq(false)); @@ -39,7 +39,7 @@ fn test_bool_accessors() { assert_that!(msg.optional_bool(), eq(false)); } -#[test] +#[googletest::test] fn test_bytes_accessors() { let mut msg = TestAllTypes::new(); // Note: even though it's named 'optional_bytes', the field is actually not @@ -59,7 +59,7 @@ fn test_bytes_accessors() { assert_that!(*msg.optional_bytes(), empty()); } -#[test] +#[googletest::test] fn test_optional_bytes_accessors() { let mut msg = TestProto3Optional::new(); assert_that!(*msg.optional_bytes(), empty()); @@ -81,7 +81,7 @@ fn test_optional_bytes_accessors() { assert_that!(msg.optional_bytes_opt(), eq(Optional::Set(&b"\xffbinary\x85non-utf8"[..]))); } -#[test] +#[googletest::test] fn test_string_accessors() { let mut msg = TestAllTypes::new(); // Note: even though it's named 'optional_string', the field is actually not @@ -101,7 +101,7 @@ fn test_string_accessors() { assert_that!(*msg.optional_string().as_bytes(), empty()); } -#[test] +#[googletest::test] fn test_optional_string_accessors() { let mut msg = TestProto3Optional::new(); assert_that!(*msg.optional_string().as_bytes(), empty()); @@ -123,7 +123,7 @@ fn test_optional_string_accessors() { assert_that!(msg.optional_string_opt(), eq(Optional::Set("".into()))); } -#[test] +#[googletest::test] fn test_nested_enum_accessors() { use test_all_types::NestedEnum; @@ -137,7 +137,7 @@ fn test_nested_enum_accessors() { assert_that!(msg.optional_nested_enum(), eq(NestedEnum::Zero)); } -#[test] +#[googletest::test] fn test_optional_nested_enum_accessors() { use test_proto3_optional::NestedEnum; @@ -154,7 +154,7 @@ fn test_optional_nested_enum_accessors() { assert_that!(msg.optional_nested_enum_opt(), eq(Optional::Set(NestedEnum::Unspecified))); } -#[test] +#[googletest::test] fn test_foreign_enum_accessors() { use unittest_proto3_rust_proto::ForeignEnum; @@ -168,7 +168,7 @@ fn test_foreign_enum_accessors() { assert_that!(msg.optional_foreign_enum(), eq(ForeignEnum::ForeignZero)); } -#[test] +#[googletest::test] fn test_oneof_accessors() { use test_all_types::OneofField::*; @@ -197,7 +197,7 @@ fn test_oneof_accessors() { assert_that!(msg.oneof_field(), matches_pattern!(not_set(_))); } -#[test] +#[googletest::test] fn test_oneof_accessors_view_long_lifetime() { use test_all_types::OneofField::*; @@ -213,7 +213,7 @@ fn test_oneof_accessors_view_long_lifetime() { assert_that!(oneof, matches_pattern!(OneofUint32(eq(7)))); } -#[test] +#[googletest::test] fn test_oneof_enum_accessors() { use unittest_proto3_rust_proto::{ test_oneof2::{Foo, FooCase, NestedEnum}, @@ -231,7 +231,7 @@ fn test_oneof_enum_accessors() { assert_that!(msg.foo_case(), matches_pattern!(FooCase::FooEnum)); } -#[test] +#[googletest::test] fn test_submsg_setter() { use test_all_types::*; @@ -244,7 +244,7 @@ fn test_submsg_setter() { assert_that!(parent.optional_nested_message().bb(), eq(7)); } -#[test] +#[googletest::test] fn test_ctype_stringpiece() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_string_piece(), eq("")); diff --git a/rust/test/shared/accessors_repeated_test.rs b/rust/test/shared/accessors_repeated_test.rs index bfec487ce9..8898d9c096 100644 --- a/rust/test/shared/accessors_repeated_test.rs +++ b/rust/test/shared/accessors_repeated_test.rs @@ -13,7 +13,7 @@ use unittest_rust_proto::{test_all_types, test_all_types::NestedMessage, TestAll macro_rules! generate_repeated_numeric_test { ($(($t: ty, $field: ident)),*) => { paste! { $( - #[test] + #[googletest::test] fn [< test_repeated_ $field _accessors >]() { let mut msg = TestAllTypes::new(); assert_that!(msg.[< repeated_ $field >](), empty()); @@ -52,7 +52,7 @@ macro_rules! generate_repeated_numeric_test { ); } - #[test] + #[googletest::test] fn [< test_repeated_ $field _set >]() { let mut msg = TestAllTypes::new(); let mut msg2 = TestAllTypes::new(); @@ -70,7 +70,7 @@ macro_rules! generate_repeated_numeric_test { ); } - #[test] + #[googletest::test] fn [< test_repeated_ $field _exact_size_iterator >]() { let mut msg = TestAllTypes::new(); let mut mutator = msg.[](); @@ -104,7 +104,7 @@ generate_repeated_numeric_test!( (f64, double) ); -#[test] +#[googletest::test] fn test_repeated_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_bool(), empty()); @@ -134,7 +134,7 @@ fn test_repeated_bool_accessors() { assert_that!(msg.repeated_bool(), each(eq(false))); } -#[test] +#[googletest::test] fn test_repeated_enum_accessors() { use test_all_types::NestedEnum; @@ -172,7 +172,7 @@ fn test_repeated_enum_accessors() { assert_that!(msg.repeated_nested_enum(), each(eq(NestedEnum::Foo))); } -#[test] +#[googletest::test] fn test_repeated_enum_set() { use test_all_types::NestedEnum; @@ -184,7 +184,7 @@ fn test_repeated_enum_set() { ); } -#[test] +#[googletest::test] fn test_repeated_bool_set() { let mut msg = TestAllTypes::new(); let mut msg2 = TestAllTypes::new(); @@ -199,7 +199,7 @@ fn test_repeated_bool_set() { assert_that!(&view.iter().collect::>(), eq(&mutator2.iter().collect::>())); } -#[test] +#[googletest::test] fn test_repeated_message() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_nested_message().len(), eq(0)); @@ -230,7 +230,7 @@ fn test_repeated_message() { assert_that!(msg2.repeated_nested_message().len(), eq(0)); } -#[test] +#[googletest::test] fn test_repeated_message_setter() { let mut msg = TestAllTypes::new(); let mut nested = NestedMessage::new(); @@ -239,7 +239,7 @@ fn test_repeated_message_setter() { assert_that!(msg.repeated_nested_message().get(0).unwrap().bb(), eq(1)); } -#[test] +#[googletest::test] fn test_repeated_strings() { let mut older_msg = TestAllTypes::new(); { @@ -274,7 +274,7 @@ fn test_repeated_strings() { assert_that!(older_msg.repeated_string(), empty()); } -#[test] +#[googletest::test] fn test_repeated_bytes() { let mut older_msg = TestAllTypes::new(); { diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index c0365b1d9e..2a7795ec06 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -15,7 +15,7 @@ use std::rc::Rc; use std::sync::Arc; use unittest_rust_proto::{test_all_types, TestAllTypes}; -#[test] +#[googletest::test] fn test_default_accessors() { let msg: TestAllTypes = Default::default(); assert_that!( @@ -40,7 +40,7 @@ fn test_default_accessors() { assert_that!(msg.default_bytes(), eq("world".as_bytes())); } -#[test] +#[googletest::test] fn test_optional_fixed32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.has_optional_fixed32(), eq(false)); @@ -58,7 +58,7 @@ fn test_optional_fixed32_accessors() { assert_that!(msg.optional_fixed32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_fixed32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_fixed32(), eq(47)); @@ -76,7 +76,7 @@ fn test_default_fixed32_accessors() { assert_that!(msg.default_fixed32_opt(), eq(Optional::Unset(47))); } -#[test] +#[googletest::test] fn test_optional_fixed64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_fixed64_opt(), eq(Optional::Unset(0))); @@ -95,7 +95,7 @@ fn test_optional_fixed64_accessors() { assert_that!(msg.optional_fixed64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_fixed64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_fixed64(), eq(48)); @@ -114,7 +114,7 @@ fn test_default_fixed64_accessors() { assert_that!(msg.default_fixed64_opt(), eq(Optional::Unset(48))); } -#[test] +#[googletest::test] fn test_optional_int32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_int32_opt(), eq(Optional::Unset(0))); @@ -133,7 +133,7 @@ fn test_optional_int32_accessors() { assert_that!(msg.optional_int32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_int32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_int32(), eq(41)); @@ -156,7 +156,7 @@ fn test_default_int32_accessors() { assert_that!(msg.default_int32_opt(), eq(Optional::Unset(41))); } -#[test] +#[googletest::test] fn test_optional_int64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_int64_opt(), eq(Optional::Unset(0))); @@ -171,7 +171,7 @@ fn test_optional_int64_accessors() { assert_that!(msg.optional_int64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_int64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_int64(), eq(42)); @@ -186,7 +186,7 @@ fn test_default_int64_accessors() { assert_that!(msg.default_int64_opt(), eq(Optional::Unset(42))); } -#[test] +#[googletest::test] fn test_optional_sint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_sint32_opt(), eq(Optional::Unset(0))); @@ -201,7 +201,7 @@ fn test_optional_sint32_accessors() { assert_that!(msg.optional_sint32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_sint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_sint32(), eq(-45)); @@ -216,7 +216,7 @@ fn test_default_sint32_accessors() { assert_that!(msg.default_sint32_opt(), eq(Optional::Unset(-45))); } -#[test] +#[googletest::test] fn test_optional_sint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_sint64_opt(), eq(Optional::Unset(0))); @@ -231,7 +231,7 @@ fn test_optional_sint64_accessors() { assert_that!(msg.optional_sint64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_sint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_sint64(), eq(46)); @@ -246,7 +246,7 @@ fn test_default_sint64_accessors() { assert_that!(msg.default_sint64_opt(), eq(Optional::Unset(46))); } -#[test] +#[googletest::test] fn test_optional_uint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_uint32_opt(), eq(Optional::Unset(0))); @@ -261,7 +261,7 @@ fn test_optional_uint32_accessors() { assert_that!(msg.optional_uint32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_uint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_uint32(), eq(43)); @@ -276,7 +276,7 @@ fn test_default_uint32_accessors() { assert_that!(msg.default_uint32_opt(), eq(Optional::Unset(43))); } -#[test] +#[googletest::test] fn test_optional_uint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_uint64_opt(), eq(Optional::Unset(0))); @@ -291,7 +291,7 @@ fn test_optional_uint64_accessors() { assert_that!(msg.optional_uint64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_uint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_uint64(), eq(44)); @@ -306,7 +306,7 @@ fn test_default_uint64_accessors() { assert_that!(msg.default_uint64_opt(), eq(Optional::Unset(44))); } -#[test] +#[googletest::test] fn test_optional_float_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_float_opt(), eq(Optional::Unset(0.0))); @@ -321,7 +321,7 @@ fn test_optional_float_accessors() { assert_that!(msg.optional_float(), eq(0.0)); } -#[test] +#[googletest::test] fn test_default_float_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_float(), eq(51.5)); @@ -336,7 +336,7 @@ fn test_default_float_accessors() { assert_that!(msg.default_float_opt(), eq(Optional::Unset(51.5))); } -#[test] +#[googletest::test] fn test_optional_double_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_double_opt(), eq(Optional::Unset(0.0))); @@ -351,7 +351,7 @@ fn test_optional_double_accessors() { assert_that!(msg.optional_double(), eq(0.0)); } -#[test] +#[googletest::test] fn test_default_double_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_double(), eq(52e3)); @@ -366,7 +366,7 @@ fn test_default_double_accessors() { assert_that!(msg.default_double_opt(), eq(Optional::Unset(52e3))); } -#[test] +#[googletest::test] fn test_optional_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_bool_opt(), eq(Optional::Unset(false))); @@ -378,7 +378,7 @@ fn test_optional_bool_accessors() { assert_that!(msg.optional_bool_opt(), eq(Optional::Unset(false))); } -#[test] +#[googletest::test] fn test_default_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_bool(), eq(true)); @@ -393,7 +393,7 @@ fn test_default_bool_accessors() { assert_that!(msg.default_bool_opt(), eq(Optional::Unset(true))); } -#[test] +#[googletest::test] fn test_optional_bytes_accessors() { let mut msg = TestAllTypes::new(); assert_that!(*msg.optional_bytes(), empty()); @@ -419,7 +419,7 @@ fn test_optional_bytes_accessors() { assert_that!(msg.optional_bytes_opt(), eq(Optional::Set(&b""[..]))); } -#[test] +#[googletest::test] fn test_into_proxied_for_bytes() { let mut msg = TestAllTypes::new(); @@ -461,7 +461,7 @@ fn test_into_proxied_for_bytes() { assert_that!(msg.optional_bytes(), eq(b"ninth")); } -#[test] +#[googletest::test] fn test_nonempty_default_bytes_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_bytes(), eq(b"world")); @@ -490,7 +490,7 @@ fn test_nonempty_default_bytes_accessors() { assert_that!(msg.default_bytes_opt(), eq(Optional::Unset(&b"world"[..]))); } -#[test] +#[googletest::test] fn test_optional_string_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_string(), eq("")); @@ -516,7 +516,7 @@ fn test_optional_string_accessors() { assert_that!(msg.optional_string_opt(), eq(Optional::Unset("".into()))); } -#[test] +#[googletest::test] fn test_into_proxied_for_string() { let mut msg = TestAllTypes::new(); @@ -565,7 +565,7 @@ fn test_into_proxied_for_string() { assert_that!(msg.optional_string(), eq("eleventh")); } -#[test] +#[googletest::test] fn test_nonempty_default_string_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_string(), eq("hello")); @@ -591,7 +591,7 @@ fn test_nonempty_default_string_accessors() { assert_that!(msg.default_string_opt(), eq(Optional::Unset("hello".into()))); } -#[test] +#[googletest::test] fn test_singular_msg_field() { let mut msg = TestAllTypes::new(); let msg_view = msg.optional_nested_message(); @@ -606,7 +606,7 @@ fn test_singular_msg_field() { assert_that!(msg.has_optional_nested_message(), eq(true)); } -#[test] +#[googletest::test] fn test_message_opt() { let msg = TestAllTypes::new(); let opt: Optional> = @@ -615,7 +615,7 @@ fn test_message_opt() { assert_that!(opt.into_inner().bb(), eq(0)); } -#[test] +#[googletest::test] fn test_message_opt_set() { let mut msg = TestAllTypes::new(); let submsg = test_all_types::NestedMessage::new(); @@ -624,7 +624,7 @@ fn test_message_opt_set() { assert_that!(msg.optional_nested_message_opt().is_set(), eq(false)); } -#[test] +#[googletest::test] fn test_setting_submsg() { let mut msg = TestAllTypes::new(); let submsg = test_all_types::NestedMessage::new(); @@ -642,7 +642,7 @@ fn test_setting_submsg() { assert_that!(msg.optional_nested_message_opt().is_set(), eq(false)); } -#[test] +#[googletest::test] fn test_msg_mut_initializes() { let mut msg = TestAllTypes::new(); assert_that!(msg.has_optional_nested_message(), eq(false)); @@ -658,7 +658,7 @@ fn test_msg_mut_initializes() { assert_that!(msg.optional_nested_message_opt().is_set(), eq(false)); } -#[test] +#[googletest::test] fn test_optional_nested_enum_accessors() { use test_all_types::NestedEnum; @@ -678,7 +678,7 @@ fn test_optional_nested_enum_accessors() { assert_that!(msg.optional_nested_enum(), eq(NestedEnum::Foo)); } -#[test] +#[googletest::test] fn test_default_nested_enum_accessors() { use test_all_types::NestedEnum; @@ -695,7 +695,7 @@ fn test_default_nested_enum_accessors() { assert_that!(msg.default_nested_enum_opt(), eq(Optional::Unset(NestedEnum::Bar))); } -#[test] +#[googletest::test] fn test_optional_foreign_enum_accessors() { use unittest_rust_proto::ForeignEnum; @@ -712,7 +712,7 @@ fn test_optional_foreign_enum_accessors() { assert_that!(msg.optional_foreign_enum(), eq(ForeignEnum::ForeignFoo)); } -#[test] +#[googletest::test] fn test_default_foreign_enum_accessors() { use unittest_rust_proto::ForeignEnum; @@ -729,7 +729,7 @@ fn test_default_foreign_enum_accessors() { assert_that!(msg.default_foreign_enum_opt(), eq(Optional::Unset(ForeignEnum::ForeignBar))); } -#[test] +#[googletest::test] fn test_optional_import_enum_accessors() { use unittest_rust_proto::ImportEnum; @@ -746,7 +746,7 @@ fn test_optional_import_enum_accessors() { assert_that!(msg.optional_import_enum(), eq(ImportEnum::ImportFoo)); } -#[test] +#[googletest::test] fn test_default_import_enum_accessors() { use unittest_rust_proto::ImportEnum; @@ -763,7 +763,7 @@ fn test_default_import_enum_accessors() { assert_that!(msg.default_import_enum_opt(), eq(Optional::Unset(ImportEnum::ImportBar))); } -#[test] +#[googletest::test] fn test_oneof_accessors() { use unittest_rust_proto::test_oneof2::{Foo::*, FooCase, NestedEnum}; use unittest_rust_proto::TestOneof2; @@ -814,7 +814,7 @@ fn test_oneof_accessors() { // TODO: Add tests covering a message-type field in a oneof. } -#[test] +#[googletest::test] fn test_msg_oneof_default_accessors() { use unittest_rust_proto::test_oneof2::{Bar::*, BarCase, NestedEnum}; @@ -846,7 +846,7 @@ fn test_msg_oneof_default_accessors() { // TODO: Add tests covering a message-type field in a oneof. } -#[test] +#[googletest::test] fn test_group() { let mut m = TestAllTypes::new(); @@ -859,7 +859,7 @@ fn test_group() { assert_that!(m.optionalgroup().a(), eq(7)); } -#[test] +#[googletest::test] fn test_submsg_setter() { use test_all_types::*; @@ -872,7 +872,7 @@ fn test_submsg_setter() { assert_that!(parent.optional_nested_message().bb(), eq(7)); } -#[test] +#[googletest::test] fn test_clone() { let mut m = TestAllTypes::new(); m.set_optional_int32(42); @@ -884,7 +884,7 @@ fn test_clone() { assert_that!(clone.optional_int32(), eq(42)); } -#[test] +#[googletest::test] fn test_to_owned() { let mut m = TestAllTypes::new(); m.set_optional_int32(42); @@ -908,7 +908,7 @@ fn test_to_owned() { assert_that!(submsg_mut.bb(), eq(8)); } -#[test] +#[googletest::test] fn test_ctype_stringpiece() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_string_piece(), eq("")); diff --git a/rust/test/shared/bad_names_test.rs b/rust/test/shared/bad_names_test.rs index 19ba5076a7..a60bfb942b 100644 --- a/rust/test/shared/bad_names_test.rs +++ b/rust/test/shared/bad_names_test.rs @@ -8,20 +8,20 @@ use bad_names_rust_proto::*; use googletest::prelude::*; -#[test] +#[googletest::test] fn test_reserved_keyword_in_accessors() { let msg = Self__mangled_because_ident_isnt_a_legal_raw_identifier::new(); let res = msg.self__mangled_because_ident_isnt_a_legal_raw_identifier().r#for(); assert_that!(res, eq(0)); } -#[test] +#[googletest::test] fn test_reserved_keyword_in_messages() { let _ = r#enum::new(); let _ = Ref::new().r#const(); } -#[test] +#[googletest::test] fn test_collision_in_accessors() { let mut m = AccessorsCollide::new(); m.set_x_mut_5(false); diff --git a/rust/test/shared/child_parent_test.rs b/rust/test/shared/child_parent_test.rs index 1a269ec052..82763b5c5c 100644 --- a/rust/test/shared/child_parent_test.rs +++ b/rust/test/shared/child_parent_test.rs @@ -7,7 +7,7 @@ use googletest::prelude::*; -#[test] +#[googletest::test] fn test_canonical_types() { let _child = child_rust_proto::Child::new(); let _parent = parent_rust_proto::Parent::new(); @@ -16,12 +16,12 @@ fn test_canonical_types() { let _parent_from_child: child_rust_proto::Parent = parent_rust_proto::Parent::new(); } -#[test] +#[googletest::test] fn test_parent_serialization() { assert_that!(*parent_rust_proto::Parent::new().serialize().unwrap(), empty()); } -#[test] +#[googletest::test] fn test_child_serialization() { assert_that!(*child_rust_proto::Child::new().serialize().unwrap(), empty()); } diff --git a/rust/test/shared/edition2023_test.rs b/rust/test/shared/edition2023_test.rs index 824e33b0be..ba31299dcc 100644 --- a/rust/test/shared/edition2023_test.rs +++ b/rust/test/shared/edition2023_test.rs @@ -11,14 +11,14 @@ use googletest::prelude::*; // is _not_ a test for Rust Edition 2023 (which doesn't exist) but instead // Protobuf Edition 2023 (which exists). -#[test] +#[googletest::test] fn check_edition2023_works() { let msg = edition2023_rust_proto::EditionsMessage::new(); assert_that!(msg.plain_field_opt().into_inner(), eq(0)); assert_that!(msg.implicit_presence_field(), eq(0)); } -#[test] +#[googletest::test] fn string_view_works() { let mut msg = edition2023_rust_proto::EditionsMessage::new(); assert_that!(msg.str_view(), eq("")); @@ -28,7 +28,7 @@ fn string_view_works() { assert_that!(msg.has_str_view(), eq(true)); } -#[test] +#[googletest::test] fn repeated_string_view_works() { let mut msg = edition2023_rust_proto::EditionsMessage::new(); assert_that!(msg.repeated_str_view().len(), eq(0)); diff --git a/rust/test/shared/enum_test.rs b/rust/test/shared/enum_test.rs index a19bcb97eb..b4d0900495 100644 --- a/rust/test/shared/enum_test.rs +++ b/rust/test/shared/enum_test.rs @@ -12,7 +12,7 @@ use googletest::prelude::*; use protobuf::Enum; use unittest_rust_proto::*; -#[test] +#[googletest::test] fn test_nested_enum_values() { assert_that!(i32::from(test_all_types::NestedEnum::Foo), eq(1)); assert_that!(i32::from(test_all_types::NestedEnum::Bar), eq(2)); @@ -20,19 +20,19 @@ fn test_nested_enum_values() { assert_that!(i32::from(test_all_types::NestedEnum::Neg), eq(-1)); } -#[test] +#[googletest::test] fn test_isolated_nested_enum() { // Ensure that the enum is generated even when it's the only nested type for the // message. assert_that!(i32::from(test_required_enum_no_mask::NestedEnum::Foo), eq(2)); } -#[test] +#[googletest::test] fn test_enum_value_name_same_as_enum() { assert_that!(i32::from(TestEnumValueNameSameAsEnum::TestEnumValueNameSameAsEnum), eq(1)); } -#[test] +#[googletest::test] fn test_enum_defaults() { assert_that!(TestSparseEnum::default(), eq(TestSparseEnum::SparseA)); assert_that!(TestEnumWithDupValue::default(), eq(TestEnumWithDupValue::Foo1)); @@ -44,7 +44,7 @@ fn test_enum_defaults() { assert_that!(test_all_types::NestedEnum::default(), eq(test_all_types::NestedEnum::Foo)); } -#[test] +#[googletest::test] #[deny(unreachable_patterns)] #[allow(clippy::let_unit_value)] fn test_closed_enum_is_nonexhaustive() { @@ -58,7 +58,7 @@ fn test_closed_enum_is_nonexhaustive() { }; } -#[test] +#[googletest::test] fn test_closed_enum_conversion() { assert_that!(i32::from(TestSparseEnum::SparseA), eq(123)); assert_that!(TestSparseEnum::try_from(123), ok(eq(&TestSparseEnum::SparseA))); @@ -70,7 +70,7 @@ fn test_closed_enum_conversion() { assert_that!(TestSparseEnum::try_from(1), err(anything())); } -#[test] +#[googletest::test] fn test_closed_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDupValue::Foo1), eq(1)); assert_that!(i32::from(TestEnumWithDupValue::Foo2), eq(1)); @@ -88,7 +88,7 @@ fn test_closed_aliased_enum_conversion() { assert_that!(TestEnumWithDupValue::Bar1, eq(TestEnumWithDupValue::Bar2)); } -#[test] +#[googletest::test] #[deny(unreachable_patterns)] #[allow(clippy::let_unit_value)] fn test_open_enum_is_nonexhaustive() { @@ -100,7 +100,7 @@ fn test_open_enum_is_nonexhaustive() { }; } -#[test] +#[googletest::test] fn test_open_enum_conversion() { assert_that!(i32::from(TestEnumWithNumericNames::Unknown), eq(0)); assert_that!(i32::from(TestEnumWithNumericNames::_2020), eq(1)); @@ -123,7 +123,7 @@ fn test_open_enum_conversion() { assert_that!(i32::from(TestEnumWithNumericNames::from(-1)), eq(-1)); } -#[test] +#[googletest::test] fn test_open_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDuplicateStrippedPrefixNames::Unknown), eq(0)); assert_that!(i32::from(TestEnumWithDuplicateStrippedPrefixNames::Foo), eq(1)); @@ -157,19 +157,19 @@ fn test_open_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDuplicateStrippedPrefixNames::from(5)), eq(5)); } -#[test] +#[googletest::test] fn test_enum_conversion_failure_display() { let err = TestSparseEnum::try_from(1).unwrap_err(); assert_that!(format!("{err}"), eq("1 is not a known value for TestSparseEnum")); } -#[test] +#[googletest::test] fn test_enum_conversion_failure_impls_std_error() { let err = TestSparseEnum::try_from(1).unwrap_err(); let _test_compiles: &dyn std::error::Error = &err; } -#[test] +#[googletest::test] fn test_is_known_for_closed_enum() { assert_that!(test_all_types::NestedEnum::is_known(-2), eq(false)); assert_that!(test_all_types::NestedEnum::is_known(-1), eq(true)); @@ -180,7 +180,7 @@ fn test_is_known_for_closed_enum() { assert_that!(test_all_types::NestedEnum::is_known(4), eq(false)); } -#[test] +#[googletest::test] fn test_is_known_for_open_enum() { assert_that!(TestEnumWithNumericNames::is_known(-1), eq(false)); assert_that!(TestEnumWithNumericNames::is_known(0), eq(true)); diff --git a/rust/test/shared/fields_with_imported_types_test.rs b/rust/test/shared/fields_with_imported_types_test.rs index 6b1b17c45b..9fa6563da1 100644 --- a/rust/test/shared/fields_with_imported_types_test.rs +++ b/rust/test/shared/fields_with_imported_types_test.rs @@ -10,7 +10,7 @@ /// a separate proto_library target. use googletest::prelude::*; -#[test] +#[googletest::test] fn test_message_field_generated() { use fields_with_imported_types_rust_proto::MsgWithFieldsWithImportedTypes; use imported_types_rust_proto::ImportedMessageView; @@ -19,7 +19,7 @@ fn test_message_field_generated() { assert_that!(msg.imported_message_field(), matches_pattern!(ImportedMessageView { .. })); } -#[test] +#[googletest::test] fn test_enum_field_generated() { use fields_with_imported_types_rust_proto::MsgWithFieldsWithImportedTypes; use imported_types_rust_proto::ImportedEnum; @@ -28,7 +28,7 @@ fn test_enum_field_generated() { assert_that!(msg.imported_enum_field(), eq(ImportedEnum::Unknown)); } -#[test] +#[googletest::test] fn test_oneof_message_field_generated() { use fields_with_imported_types_rust_proto::msg_with_fields_with_imported_types::ImportedTypesOneof::not_set; use fields_with_imported_types_rust_proto::MsgWithFieldsWithImportedTypes; diff --git a/rust/test/shared/import_public_test.rs b/rust/test/shared/import_public_test.rs index 4d05ad8f02..6299219151 100644 --- a/rust/test/shared/import_public_test.rs +++ b/rust/test/shared/import_public_test.rs @@ -7,7 +7,7 @@ //! Tests covering codegen of import public statements. -#[test] +#[googletest::test] fn test_import_public_types_are_reexported() { let _: import_public_rust_proto::PrimarySrcPubliclyImportedMsg; let _: import_public_rust_proto::PrimarySrcPubliclyImportedMsgView; diff --git a/rust/test/shared/merge_from_test.rs b/rust/test/shared/merge_from_test.rs index b07f528fd9..26eb3998ed 100644 --- a/rust/test/shared/merge_from_test.rs +++ b/rust/test/shared/merge_from_test.rs @@ -2,7 +2,7 @@ use googletest::prelude::*; use protobuf::proto; use unittest_rust_proto::{NestedTestAllTypes, TestAllTypes}; -#[test] +#[googletest::test] fn merge_from_empty() { let mut dst = TestAllTypes::new(); let src = TestAllTypes::new(); @@ -10,7 +10,7 @@ fn merge_from_empty() { assert_that!(dst.as_view().has_optional_int32(), eq(false)); } -#[test] +#[googletest::test] fn merge_from_non_empty() { let mut dst = TestAllTypes::new(); let src = proto!(TestAllTypes { optional_int32: 42 }); @@ -18,7 +18,7 @@ fn merge_from_non_empty() { assert_eq!(dst.as_view().optional_int32(), 42); } -#[test] +#[googletest::test] fn merge_repeated_empty() { let mut dst = TestAllTypes::new(); let mut src = TestAllTypes::new(); @@ -30,7 +30,7 @@ fn merge_repeated_empty() { ); } -#[test] +#[googletest::test] fn merge_repeated_non_empty() { let mut dst = TestAllTypes::new(); let mut src = TestAllTypes::new(); @@ -43,7 +43,7 @@ fn merge_repeated_non_empty() { ); } -#[test] +#[googletest::test] fn merge_from_sub_message() { let mut dst = NestedTestAllTypes::new(); let src = proto!(NestedTestAllTypes { diff --git a/rust/test/shared/nested_types_test.rs b/rust/test/shared/nested_types_test.rs index d47929efb4..8f04d9f38d 100644 --- a/rust/test/shared/nested_types_test.rs +++ b/rust/test/shared/nested_types_test.rs @@ -7,7 +7,7 @@ //! Tests covering nested types. -#[test] +#[googletest::test] fn test_nested_messages_accessible() { let _parent: unittest_rust_proto::TestAllTypes; let _child: unittest_rust_proto::test_all_types::NestedMessage; @@ -15,7 +15,7 @@ fn test_nested_messages_accessible() { nested_test_all_extensions_data::NestedDynamicExtensions::new(); } -#[test] +#[googletest::test] fn test_nested_enums_accessible() { let _parent: unittest_rust_proto::TestAllTypes; let _child: unittest_rust_proto::test_all_types::NestedEnum; diff --git a/rust/test/shared/package_test.rs b/rust/test/shared/package_test.rs index 04f5b84d5f..67a854d6d3 100644 --- a/rust/test/shared/package_test.rs +++ b/rust/test/shared/package_test.rs @@ -7,7 +7,7 @@ //! Tests covering proto packages. -#[test] +#[googletest::test] fn test_message_packages() { // empty package, message declared in the first .proto source let _: no_package_rust_proto::MsgWithoutPackage; @@ -26,7 +26,7 @@ fn test_message_packages() { let _: package_rust_proto::ImportedMsgWithPackage; } -#[test] +#[googletest::test] fn test_enum_packages() { // empty package, enum declared in the first .proto source let _: no_package_rust_proto::EnumWithoutPackage; diff --git a/rust/test/shared/proto_macro_test.rs b/rust/test/shared/proto_macro_test.rs index 876a5a47f4..5085cb73f5 100644 --- a/rust/test/shared/proto_macro_test.rs +++ b/rust/test/shared/proto_macro_test.rs @@ -18,7 +18,7 @@ struct TestValue { val: i64, } -#[test] +#[googletest::test] fn test_setting_literals() { let fixed64 = || 108; let test_ref = |x: &i64| *x; @@ -67,7 +67,7 @@ fn test_setting_literals() { assert_that!(msg.optional_nested_enum(), eq(test_all_types::NestedEnum::Baz)); } -#[test] +#[googletest::test] fn single_nested_message() { let msg = proto!(TestAllTypes { optional_nested_message: NestedMessage { bb: 42 } }); assert_that!(msg.optional_nested_message().bb(), eq(42)); @@ -122,7 +122,7 @@ fn single_nested_message() { assert_that!(msg.has_optional_nested_message(), eq(true)); } -#[test] +#[googletest::test] fn test_recursive_msg() { let msg = proto!(NestedTestAllTypes { child: NestedTestAllTypes { @@ -139,7 +139,7 @@ fn test_recursive_msg() { assert_that!(msg.child().child().child().payload().optional_int32(), eq(43)); } -#[test] +#[googletest::test] fn test_spread_msg() { let msg = proto!(TestAllTypes { optional_nested_message: NestedMessage { bb: 42 } }); let msg2 = proto!(TestAllTypes { ..msg.as_view() }); @@ -149,7 +149,7 @@ fn test_spread_msg() { assert_that!(msg3.optional_int32(), eq(1)); } -#[test] +#[googletest::test] fn test_spread_nested_msg() { let msg = proto!(NestedTestAllTypes { child: NestedTestAllTypes { @@ -168,7 +168,7 @@ fn test_spread_nested_msg() { assert_that!(msg2.child().child().child().payload().optional_int32(), eq(43)); } -#[test] +#[googletest::test] fn test_repeated_i32() { let msg = proto!(TestAllTypes { repeated_int32: [1, 1 + 1, 3] }); assert_that!(msg.repeated_int32().len(), eq(3)); @@ -177,7 +177,7 @@ fn test_repeated_i32() { assert_that!(msg.repeated_int32().get(2).unwrap(), eq(3)); } -#[test] +#[googletest::test] fn test_repeated_msg() { let msg2 = proto!(NestedTestAllTypes { payload: TestAllTypes { optional_int32: 1 } }); let msg = proto!(NestedTestAllTypes { diff --git a/rust/test/shared/serialization_test.rs b/rust/test/shared/serialization_test.rs index e5b614034f..bf3d67a0b5 100644 --- a/rust/test/shared/serialization_test.rs +++ b/rust/test/shared/serialization_test.rs @@ -15,7 +15,7 @@ use unittest_rust_proto::TestAllTypes as TestAllTypesProto2; macro_rules! generate_parameterized_serialization_test { ($(($type: ident, $name_ext: ident)),*) => { paste! { $( - #[test] + #[googletest::test] fn [< serialization_zero_length_ $name_ext >]() { let mut msg = [< $type >]::new(); @@ -29,7 +29,7 @@ macro_rules! generate_parameterized_serialization_test { assert_that!(serialized.len(), eq(0)); } - #[test] + #[googletest::test] fn [< serialize_deserialize_message_ $name_ext>]() { let mut msg = [< $type >]::new(); msg.set_optional_int64(42); @@ -44,17 +44,17 @@ macro_rules! generate_parameterized_serialization_test { assert_that!(msg.optional_bytes(), eq(msg2.optional_bytes())); } - #[test] + #[googletest::test] fn [< deserialize_empty_ $name_ext>]() { assert!([< $type >]::parse(&[]).is_ok()); } - #[test] + #[googletest::test] fn [< deserialize_error_ $name_ext>]() { assert!([< $type >]::parse(b"not a serialized proto").is_err()); } - #[test] + #[googletest::test] fn [< set_bytes_with_serialized_data_ $name_ext>]() { let mut msg = [< $type >]::new(); msg.set_optional_int64(42); @@ -64,7 +64,7 @@ macro_rules! generate_parameterized_serialization_test { assert_that!(msg2.optional_bytes(), eq(msg.serialize().unwrap())); } - #[test] + #[googletest::test] fn [< deserialize_on_previously_allocated_message_ $name_ext>]() { let mut msg = [< $type >]::new(); msg.set_optional_int64(42); @@ -95,7 +95,7 @@ macro_rules! generate_parameterized_int32_byte_size_test { ($(($type: ident, $name_ext: ident)),*) => { paste! { $( - #[test] + #[googletest::test] fn [< test_int32_byte_size_ $name_ext>]() { let args = vec![(0, 1), (127, 1), (128, 2), (-1, 10)]; for arg in args { diff --git a/rust/test/shared/simple_nested_test.rs b/rust/test/shared/simple_nested_test.rs index fe74786ec8..2fefc466de 100644 --- a/rust/test/shared/simple_nested_test.rs +++ b/rust/test/shared/simple_nested_test.rs @@ -10,7 +10,7 @@ use nested_rust_proto::outer::inner::InnerEnum; use nested_rust_proto::outer::InnerView; use nested_rust_proto::*; -#[test] +#[googletest::test] fn test_deeply_nested_message() { let deep = outer::inner::super_inner::duper_inner::even_more_inner::CantBelieveItsSoInner::new(); @@ -20,7 +20,7 @@ fn test_deeply_nested_message() { assert_that!(outermsg.deep().num(), eq(0)); } -#[test] +#[googletest::test] fn test_deeply_nested_enum() { use outer::inner::super_inner::duper_inner::even_more_inner::JustWayTooInner; let deep = JustWayTooInner::default(); @@ -30,7 +30,7 @@ fn test_deeply_nested_enum() { assert_that!(outermsg.deep_enum(), eq(JustWayTooInner::Unspecified)); } -#[test] +#[googletest::test] fn test_nested_views() { let outermsg = Outer::new(); let inner_msg: InnerView<'_> = outermsg.inner(); @@ -53,7 +53,7 @@ fn test_nested_views() { assert_that!(inner_msg.inner_enum(), eq(InnerEnum::Unspecified)); } -#[test] +#[googletest::test] fn test_nested_view_lifetimes() { // Ensure that views have the lifetime of the first layer of borrow, and don't // create intermediate borrows through nested accessors. @@ -79,7 +79,7 @@ fn test_nested_view_lifetimes() { assert_that!(string_map.len(), eq(0)); } -#[test] +#[googletest::test] fn test_msg_from_outside() { // let's make sure that we're not just working for messages nested inside // messages, messages from without and within should work @@ -87,7 +87,7 @@ fn test_msg_from_outside() { assert_that!(outer.notinside().num(), eq(0)); } -#[test] +#[googletest::test] fn test_recursive_view() { let rec = nested_rust_proto::Recursive::new(); assert_that!(rec.num(), eq(0)); @@ -100,7 +100,7 @@ fn test_recursive_view() { assert_that!(nested.num(), eq(0)); } -#[test] +#[googletest::test] fn test_recursive_mut() { let mut rec = nested_rust_proto::Recursive::new(); let mut one = rec.rec_mut(); diff --git a/rust/test/shared/utf8/utf8_test.rs b/rust/test/shared/utf8/utf8_test.rs index 8b46cf86dd..e4431efbe3 100644 --- a/rust/test/shared/utf8/utf8_test.rs +++ b/rust/test/shared/utf8/utf8_test.rs @@ -34,7 +34,7 @@ fn make_non_utf8_proto_str() -> &'static ProtoStr { } } -#[test] +#[googletest::test] fn test_proto2() { let non_utf8_str = make_non_utf8_proto_str(); @@ -52,7 +52,7 @@ fn test_proto2() { assert_that!(parsed_result, ok(anything())); } -#[test] +#[googletest::test] fn test_proto3() { let non_utf8_str = make_non_utf8_proto_str(); @@ -70,7 +70,7 @@ fn test_proto3() { assert_that!(parsed_result, err(matches_pattern!(&ParseError))); } -#[test] +#[googletest::test] fn test_verify() { let non_utf8_str = make_non_utf8_proto_str(); diff --git a/rust/test/upb/debug_string_test.rs b/rust/test/upb/debug_string_test.rs index fa2f429c1c..9cc3457f1e 100644 --- a/rust/test/upb/debug_string_test.rs +++ b/rust/test/upb/debug_string_test.rs @@ -13,7 +13,7 @@ use unittest_rust_proto::{ test_all_types::NestedMessage as NestedMessageProto2, TestAllTypes as TestAllTypesProto2, }; -#[test] +#[googletest::test] fn test_debug_string() { let mut msg = proto!(TestAllTypesProto2 { optional_int32: 42, diff --git a/rust/test/upb/string_ctypes_test.rs b/rust/test/upb/string_ctypes_test.rs index 6bbea67a21..8296592b0c 100644 --- a/rust/test/upb/string_ctypes_test.rs +++ b/rust/test/upb/string_ctypes_test.rs @@ -8,7 +8,7 @@ use googletest::prelude::*; use unittest_proto3_rust_proto::TestAllTypes; -#[test] +#[googletest::test] fn test_stringpiece_repeated() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_string_piece().len(), eq(0)); @@ -17,7 +17,7 @@ fn test_stringpiece_repeated() { assert_that!(msg.repeated_string_piece().get(0), some(eq("hello"))); } -#[test] +#[googletest::test] fn test_cord() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_cord(), eq("")); @@ -25,7 +25,7 @@ fn test_cord() { assert_that!(msg.optional_cord(), eq("hello")); } -#[test] +#[googletest::test] fn test_cord_repeated() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_cord().len(), eq(0)); From d0395408aab58974e8323bb92273034c2ecd7a56 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 07:24:22 -0700 Subject: [PATCH 040/107] Fix outputted command for usage of update_failure_list.py PiperOrigin-RevId: 654738180 --- conformance/BUILD.bazel | 5 ++ conformance/conformance_test.cc | 54 +++++++++++-------- .../protobuf/internal/api_implementation.cc | 2 +- .../protobuf/compiler/python/pyi_generator.cc | 2 +- upb/wire/eps_copy_input_stream_test.cc | 2 +- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index 384bb56563..5a6c271856 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -304,6 +304,11 @@ py_binary( ], ) +py_binary( + name = "update_failure_list", + srcs = ["update_failure_list.py"], +) + inline_sh_binary( name = "conformance_php", testonly = 1, diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 2d7dfcd344..0f584b95a3 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -70,9 +70,6 @@ bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file, absl::string_view filename = write_to_file; if (!output_dir.empty()) { full_filename = std::string(output_dir); - if (*output_dir.rbegin() != '/') { - full_filename.push_back('/'); - } absl::StrAppend(&full_filename, write_to_file); filename = full_filename; } @@ -82,7 +79,9 @@ bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file, os << v << "\n"; } } else { - absl::StrAppendFormat(output, "Failed to open file: %s\n", filename); + absl::StrAppendFormat(output, + "Failed to open file: %s\n", + filename); } } @@ -468,37 +467,48 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, } RunSuiteImpl(); + if (*output_dir_.rbegin() != '/') { + output_dir_.push_back('/'); + } + bool ok = true; if (!CheckSetEmpty( expected_to_fail_, "nonexistent_tests.txt", - absl::StrCat("These tests were listed in the failure list, but they " - "don't exist. Remove them from the failure list by " - "running:\n" - " ./update_failure_list.py ", - failure_list_filename_, - " --remove nonexistent_tests.txt"), + absl::StrCat( + "These tests were listed in the failure list, but they " + "don't exist. Remove them from the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --remove ", output_dir_, + "nonexistent_tests.txt"), output_dir_, &output_)) { ok = false; } if (!CheckSetEmpty( unexpected_failing_tests_, "failing_tests.txt", - absl::StrCat("These tests failed. If they can't be fixed right now, " - "you can add them to the failure list so the overall " - "suite can succeed. Add them to the failure list by " - "running:\n" - " ./update_failure_list.py ", - failure_list_filename_, " --add failing_tests.txt"), + absl::StrCat( + "These tests failed. If they can't be fixed right now, " + "you can add them to the failure list so the overall " + "suite can succeed. Add them to the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --add ", output_dir_, + "failing_tests.txt"), output_dir_, &output_)) { ok = false; } if (!CheckSetEmpty( unexpected_succeeding_tests_, "succeeding_tests.txt", - absl::StrCat("These tests succeeded, even though they were listed in " - "the failure list. Remove them from the failure list " - "by running:\n" - " ./update_failure_list.py ", - failure_list_filename_, - " --remove succeeding_tests.txt"), + absl::StrCat( + "These tests succeeded, even though they were listed in " + "the failure list. Remove them from the failure list by running " + "from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --remove ", output_dir_, + "succeeding_tests.txt"), output_dir_, &output_)) { ok = false; } diff --git a/python/google/protobuf/internal/api_implementation.cc b/python/google/protobuf/internal/api_implementation.cc index e32f4acb68..06eb05f706 100644 --- a/python/google/protobuf/internal/api_implementation.cc +++ b/python/google/protobuf/internal/api_implementation.cc @@ -51,7 +51,7 @@ static const char kModuleDocstring[] = "\n" "It complements api_implementation.py by setting defaults using compile-time\n" "constants defined in C, such that one can set defaults at compilation\n" -"(e.g. with blaze flag --copt=-DPYTHON_PROTO2_CPP_IMPL_V2)."; +"(e.g. with bazel flag --copt=-DPYTHON_PROTO2_CPP_IMPL_V2)."; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef _module = { diff --git a/src/google/protobuf/compiler/python/pyi_generator.cc b/src/google/protobuf/compiler/python/pyi_generator.cc index 323f93eba4..f86777c2c2 100644 --- a/src/google/protobuf/compiler/python/pyi_generator.cc +++ b/src/google/protobuf/compiler/python/pyi_generator.cc @@ -567,7 +567,7 @@ bool PyiGenerator::Generate(const FileDescriptor* file, import_map_.clear(); // Calculate file name. file_ = file; - // In google3, devtools/python/blaze/pytype/pytype_impl.bzl uses --pyi_out to + // In google3, devtools/python/bazel/pytype/pytype_impl.bzl uses --pyi_out to // directly set the output file name. std::vector > options; ParseGeneratorParameter(parameter, &options); diff --git a/upb/wire/eps_copy_input_stream_test.cc b/upb/wire/eps_copy_input_stream_test.cc index c28d1457e4..42664748e3 100644 --- a/upb/wire/eps_copy_input_stream_test.cc +++ b/upb/wire/eps_copy_input_stream_test.cc @@ -281,7 +281,7 @@ TEST(EpsCopyInputStreamTest, ZeroSize) { // } // // // Test with: -// // $ blaze run --config=fuzztest third_party/upb:eps_copy_input_stream_test \ +// // $ bazel run --config=fuzztest third_party/upb:eps_copy_input_stream_test \ // // -- --gunit_fuzz= // FUZZ_TEST(EpsCopyFuzzTest, TestAgainstFakeStream) // .WithDomains(ArbitraryEpsCopyTestScript()); From 935783570eb82957d493164a4f3e17cf254e38e5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 07:40:02 -0700 Subject: [PATCH 041/107] Add arena support to UnknownFieldSet: - Now it is arena constructible. Allocates the array, strings, and inner sets in the arena. - Now it is arena destructible. Will skip destructors when in an arena. Also, optimize `default_instance()` now that we can provide a `constinit` version of it. PiperOrigin-RevId: 654743861 --- src/google/protobuf/internal_visibility.h | 2 + src/google/protobuf/repeated_field.h | 2 + src/google/protobuf/unknown_field_set.cc | 116 +++++------- src/google/protobuf/unknown_field_set.h | 179 +++++++++++------- .../protobuf/unknown_field_set_unittest.cc | 108 ++++++++++- 5 files changed, 261 insertions(+), 146 deletions(-) diff --git a/src/google/protobuf/internal_visibility.h b/src/google/protobuf/internal_visibility.h index fed45bd944..fd9bfc4115 100644 --- a/src/google/protobuf/internal_visibility.h +++ b/src/google/protobuf/internal_visibility.h @@ -17,6 +17,7 @@ class MessageLite; namespace internal { class InternalVisibilityForTesting; +class InternalMetadata; // Empty class to use as a mandatory 'internal token' for functions that have to // be public, such as arena constructors, but that are for internal use only. @@ -29,6 +30,7 @@ class InternalVisibility { friend class ::google::protobuf::Arena; friend class ::google::protobuf::Message; friend class ::google::protobuf::MessageLite; + friend class ::google::protobuf::internal::InternalMetadata; friend class InternalVisibilityForTesting; }; diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 027c80e353..c34746cbd0 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -56,6 +56,7 @@ namespace google { namespace protobuf { class Message; +class UnknownField; // For the allowlist namespace internal { @@ -141,6 +142,7 @@ class RepeatedField final absl::disjunction, internal::is_supported_floating_point_type, std::is_same, + std::is_same, is_proto_enum>::value, "We only support non-string scalars in RepeatedField."); } diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index 74fbb8f1b2..ecf66f5e82 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -16,7 +16,6 @@ #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/string_view.h" #include "google/protobuf/extension_set.h" -#include "google/protobuf/generated_message_tctable_decl.h" #include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream.h" @@ -32,38 +31,23 @@ namespace google { namespace protobuf { -const UnknownFieldSet& UnknownFieldSet::default_instance() { - static auto instance = internal::OnShutdownDelete(new UnknownFieldSet()); - return *instance; -} - void UnknownFieldSet::ClearFallback() { ABSL_DCHECK(!fields_.empty()); - int n = fields_.size(); - do { - (fields_)[--n].Delete(); - } while (n > 0); - fields_.clear(); -} - -void UnknownFieldSet::InternalMergeFrom(const UnknownFieldSet& other) { - int other_field_count = other.field_count(); - if (other_field_count > 0) { - fields_.reserve(fields_.size() + other_field_count); - for (int i = 0; i < other_field_count; i++) { - fields_.push_back((other.fields_)[i]); - fields_.back().DeepCopy((other.fields_)[i]); - } + if (arena() == nullptr) { + int n = fields_.size(); + do { + (fields_)[--n].Delete(); + } while (n > 0); } + fields_.Clear(); } void UnknownFieldSet::MergeFrom(const UnknownFieldSet& other) { int other_field_count = other.field_count(); if (other_field_count > 0) { - fields_.reserve(fields_.size() + other_field_count); - for (int i = 0; i < other_field_count; i++) { - fields_.push_back((other.fields_)[i]); - fields_.back().DeepCopy((other.fields_)[i]); + fields_.Reserve(fields_.size() + other_field_count); + for (auto elem : other.fields_) { + fields_.Add(elem.DeepCopy(arena())); } } } @@ -71,14 +55,14 @@ void UnknownFieldSet::MergeFrom(const UnknownFieldSet& other) { // A specialized MergeFrom for performance when we are merging from an UFS that // is temporary and can be destroyed in the process. void UnknownFieldSet::MergeFromAndDestroy(UnknownFieldSet* other) { - if (fields_.empty()) { - fields_ = std::move(other->fields_); + if (arena() != other->arena()) { + MergeFrom(*other); + } else if (fields_.empty()) { + fields_.Swap(&other->fields_); } else { - fields_.insert(fields_.end(), - std::make_move_iterator(other->fields_.begin()), - std::make_move_iterator(other->fields_.end())); + fields_.MergeFrom(other->fields_); + other->fields_.Clear(); } - other->fields_.clear(); } void UnknownFieldSet::MergeToInternalMetadata( @@ -89,7 +73,7 @@ void UnknownFieldSet::MergeToInternalMetadata( size_t UnknownFieldSet::SpaceUsedExcludingSelfLong() const { if (fields_.empty()) return 0; - size_t total_size = sizeof(UnknownField) * fields_.capacity(); + size_t total_size = fields_.SpaceUsedExcludingSelfLong(); for (const UnknownField& field : fields_) { switch (field.type()) { @@ -113,65 +97,54 @@ size_t UnknownFieldSet::SpaceUsedLong() const { } void UnknownFieldSet::AddVarint(int number, uint64_t value) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_VARINT); field.data_.varint_ = value; } void UnknownFieldSet::AddFixed32(int number, uint32_t value) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_FIXED32); field.data_.fixed32_ = value; } void UnknownFieldSet::AddFixed64(int number, uint64_t value) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_FIXED64); field.data_.fixed64_ = value; } std::string* UnknownFieldSet::AddLengthDelimited(int number) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); - field.data_.string_value = new std::string; + field.data_.string_value = Arena::Create(arena()); return field.data_.string_value; } UnknownFieldSet* UnknownFieldSet::AddGroup(int number) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_GROUP); - field.data_.group_ = new UnknownFieldSet; + field.data_.group_ = Arena::Create(arena()); return field.data_.group_; } void UnknownFieldSet::AddField(const UnknownField& field) { - fields_.push_back(field); - fields_.back().DeepCopy(field); + fields_.Add(field.DeepCopy(arena())); } void UnknownFieldSet::DeleteSubrange(int start, int num) { - // Delete the specified fields. - for (int i = 0; i < num; ++i) { - (fields_)[i + start].Delete(); - } - // Slide down the remaining fields. - for (size_t i = start + num; i < fields_.size(); ++i) { - (fields_)[i - num] = (fields_)[i]; - } - // Pop off the # of deleted fields. - for (int i = 0; i < num; ++i) { - fields_.pop_back(); + if (arena() == nullptr) { + // Delete the specified fields. + for (int i = 0; i < num; ++i) { + (fields_)[i + start].Delete(); + } } + fields_.ExtractSubrange(start, num, nullptr); } void UnknownFieldSet::DeleteByNumber(int number) { @@ -179,7 +152,9 @@ void UnknownFieldSet::DeleteByNumber(int number) { for (size_t i = 0; i < fields_.size(); ++i) { UnknownField* field = &(fields_)[i]; if (field->number() == number) { - field->Delete(); + if (arena() == nullptr) { + field->Delete(); + } } else { if (i != left) { (fields_)[left] = (fields_)[i]; @@ -187,7 +162,7 @@ void UnknownFieldSet::DeleteByNumber(int number) { ++left; } } - fields_.resize(left); + fields_.Truncate(left); } bool UnknownFieldSet::MergeFromCodedStream(io::CodedInputStream* input) { @@ -257,21 +232,32 @@ void UnknownField::Delete() { } } -void UnknownField::DeepCopy(const UnknownField& other) { - (void)other; // Parameter is used by Google-internal code. +UnknownField UnknownField::DeepCopy(Arena* arena) const { + UnknownField copy = *this; switch (type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - data_.string_value = new std::string(*data_.string_value); + copy.data_.string_value = + Arena::Create(arena, *data_.string_value); break; case UnknownField::TYPE_GROUP: { - UnknownFieldSet* group = new UnknownFieldSet(); - group->InternalMergeFrom(*data_.group_); - data_.group_ = group; + UnknownFieldSet* group = Arena::Create(arena); + group->MergeFrom(*data_.group_); + copy.data_.group_ = group; break; } default: break; } + return copy; +} + +void UnknownFieldSet::SwapSlow(UnknownFieldSet* other) { + UnknownFieldSet tmp; + tmp.MergeFrom(*this); + this->Clear(); + this->MergeFrom(*other); + other->Clear(); + other->MergeFrom(tmp); } uint8_t* UnknownField::InternalSerializeLengthDelimitedNoTag( diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index a58c36478d..e4cdb70a9a 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -19,17 +19,19 @@ #include #include -#include #include "google/protobuf/stubs/common.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "google/protobuf/arena.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/message_lite.h" +#include "google/protobuf/metadata_lite.h" #include "google/protobuf/parse_context.h" #include "google/protobuf/port.h" +#include "google/protobuf/repeated_field.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -54,7 +56,67 @@ using UFSStringView = const std::string&; } // namespace internal class Message; // message.h -class UnknownField; // below + +// Represents one field in an UnknownFieldSet. +class PROTOBUF_EXPORT UnknownField { + public: + enum Type { + TYPE_VARINT, + TYPE_FIXED32, + TYPE_FIXED64, + TYPE_LENGTH_DELIMITED, + TYPE_GROUP + }; + + // The field's field number, as seen on the wire. + inline int number() const; + + // The field type. + inline Type type() const; + + // Accessors ------------------------------------------------------- + // Each method works only for UnknownFields of the corresponding type. + + inline uint64_t varint() const; + inline uint32_t fixed32() const; + inline uint64_t fixed64() const; + inline internal::UFSStringView length_delimited() const; + inline const UnknownFieldSet& group() const; + + inline void set_varint(uint64_t value); + inline void set_fixed32(uint32_t value); + inline void set_fixed64(uint64_t value); + inline void set_length_delimited(absl::string_view value); + inline std::string* mutable_length_delimited(); + inline UnknownFieldSet* mutable_group(); + + inline size_t GetLengthDelimitedSize() const; + uint8_t* InternalSerializeLengthDelimitedNoTag( + uint8_t* target, io::EpsCopyOutputStream* stream) const; + + private: + friend class UnknownFieldSet; + + // If this UnknownField contains a pointer, delete it. + void Delete(); + + // Make a deep copy of any pointers in this UnknownField. + UnknownField DeepCopy(Arena* arena) const; + + // Set the wire type of this UnknownField. Should only be used when this + // UnknownField is being created. + inline void SetType(Type type); + + uint32_t number_; + uint32_t type_; + union { + uint64_t varint_; + uint32_t fixed32_; + uint64_t fixed64_; + std::string* string_value; + UnknownFieldSet* group_; + } data_; +}; // An UnknownFieldSet contains fields that were encountered while parsing a // message but were not defined by its type. Keeping track of these can be @@ -70,7 +132,7 @@ class UnknownField; // below // the Reflection interface which is independent of any serialization scheme. class PROTOBUF_EXPORT UnknownFieldSet { public: - UnknownFieldSet(); + constexpr UnknownFieldSet(); UnknownFieldSet(const UnknownFieldSet&) = delete; UnknownFieldSet& operator=(const UnknownFieldSet&) = delete; ~UnknownFieldSet(); @@ -167,13 +229,20 @@ class PROTOBUF_EXPORT UnknownFieldSet { bool SerializeToCodedStream(io::CodedOutputStream* output) const; static const UnknownFieldSet& default_instance(); + UnknownFieldSet(internal::InternalVisibility, Arena* arena) + : UnknownFieldSet(arena) {} + private: - // For InternalMergeFrom - friend class UnknownField; - // Merges from other UnknownFieldSet. This method assumes, that this object - // is newly created and has no fields. - void InternalMergeFrom(const UnknownFieldSet& other); + using InternalArenaConstructable_ = void; + using DestructorSkippable_ = void; + + friend class google::protobuf::Arena; + explicit UnknownFieldSet(Arena* arena) : fields_(arena) {} + + Arena* arena() { return fields_.GetArena(); } + void ClearFallback(); + void SwapSlow(UnknownFieldSet* other); template fields_; + RepeatedField fields_; }; namespace internal { @@ -218,74 +287,19 @@ const char* UnknownFieldParse(uint64_t tag, UnknownFieldSet* unknown, } // namespace internal -// Represents one field in an UnknownFieldSet. -class PROTOBUF_EXPORT UnknownField { - public: - enum Type { - TYPE_VARINT, - TYPE_FIXED32, - TYPE_FIXED64, - TYPE_LENGTH_DELIMITED, - TYPE_GROUP - }; - - // The field's field number, as seen on the wire. - inline int number() const; - - // The field type. - inline Type type() const; - - // Accessors ------------------------------------------------------- - // Each method works only for UnknownFields of the corresponding type. - - inline uint64_t varint() const; - inline uint32_t fixed32() const; - inline uint64_t fixed64() const; - inline internal::UFSStringView length_delimited() const; - inline const UnknownFieldSet& group() const; - - inline void set_varint(uint64_t value); - inline void set_fixed32(uint32_t value); - inline void set_fixed64(uint64_t value); - inline void set_length_delimited(absl::string_view value); - inline std::string* mutable_length_delimited(); - inline UnknownFieldSet* mutable_group(); - - inline size_t GetLengthDelimitedSize() const; - uint8_t* InternalSerializeLengthDelimitedNoTag( - uint8_t* target, io::EpsCopyOutputStream* stream) const; - - private: - friend class UnknownFieldSet; - - // If this UnknownField contains a pointer, delete it. - void Delete(); - - // Make a deep copy of any pointers in this UnknownField. - void DeepCopy(const UnknownField& other); - - // Set the wire type of this UnknownField. Should only be used when this - // UnknownField is being created. - inline void SetType(Type type); - - uint32_t number_; - uint32_t type_; - union { - uint64_t varint_; - uint32_t fixed32_; - uint64_t fixed64_; - std::string* string_value; - UnknownFieldSet* group_; - } data_; -}; - // =================================================================== // inline implementations -inline UnknownFieldSet::UnknownFieldSet() {} +constexpr UnknownFieldSet::UnknownFieldSet() = default; inline UnknownFieldSet::~UnknownFieldSet() { Clear(); } +inline const UnknownFieldSet& UnknownFieldSet::default_instance() { + PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT static const UnknownFieldSet + instance; + return instance; +} + inline void UnknownFieldSet::ClearAndFreeMemory() { Clear(); } inline void UnknownFieldSet::Clear() { @@ -297,7 +311,12 @@ inline void UnknownFieldSet::Clear() { inline bool UnknownFieldSet::empty() const { return fields_.empty(); } inline void UnknownFieldSet::Swap(UnknownFieldSet* x) { - fields_.swap(x->fields_); + if (arena() == x->arena()) { + fields_.Swap(&x->fields_); + } else { + // We might need to do a deep copy, so use Merge instead + SwapSlow(x); + } } inline int UnknownFieldSet::field_count() const { @@ -378,6 +397,22 @@ inline size_t UnknownField::GetLengthDelimitedSize() const { inline void UnknownField::SetType(Type type) { type_ = type; } +namespace internal { + +// Add specialization of InternalMetadata::Container to provide arena support. +template <> +struct InternalMetadata::Container + : public InternalMetadata::ContainerBase { + UnknownFieldSet unknown_fields; + + explicit Container(Arena* input_arena) + : unknown_fields(InternalVisibility{}, input_arena) {} + + using InternalArenaConstructable_ = void; + using DestructorSkippable_ = void; +}; +} // namespace internal + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/unknown_field_set_unittest.cc b/src/google/protobuf/unknown_field_set_unittest.cc index 4e926123d6..27923675f1 100644 --- a/src/google/protobuf/unknown_field_set_unittest.cc +++ b/src/google/protobuf/unknown_field_set_unittest.cc @@ -175,6 +175,94 @@ TEST_F(UnknownFieldSetTest, Group) { EXPECT_EQ(all_fields_.optionalgroup().a(), nested_field.varint()); } +static void PopulateUFS(UnknownFieldSet& set) { + UnknownFieldSet* node = &set; + for (int i = 0; i < 3; ++i) { + node->AddVarint(1, 100); + const char* long_str = "This is a very long string, not sso"; + node->AddLengthDelimited(2, long_str); + *node->AddLengthDelimited(3) = long_str; + // Test some recursion too. + node = node->AddGroup(4); + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithMergeFrom) { + Arena arena; + + for (bool lhs_arena : {false, true}) { + for (bool rhs_arena : {false, true}) { + UnknownFieldSet lhs_stack, rhs_stack; + auto& lhs = + lhs_arena ? *Arena::Create(&arena) : lhs_stack; + auto& rhs = + rhs_arena ? *Arena::Create(&arena) : rhs_stack; + PopulateUFS(rhs); + lhs.MergeFrom(rhs); + } + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithMergeAndDestroy) { + Arena arena; + + for (bool lhs_arena : {false, true}) { + for (bool populate_lhs : {false, true}) { + for (bool rhs_arena : {false, true}) { + for (bool populate_rhs : {false, true}) { + UnknownFieldSet lhs_stack, rhs_stack; + auto& lhs = + lhs_arena ? *Arena::Create(&arena) : lhs_stack; + auto& rhs = + rhs_arena ? *Arena::Create(&arena) : rhs_stack; + if (populate_lhs) PopulateUFS(lhs); + if (populate_rhs) PopulateUFS(rhs); + lhs.MergeFromAndDestroy(&rhs); + } + } + } + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithSwap) { + Arena arena; + + for (bool lhs_arena : {false, true}) { + for (bool rhs_arena : {false, true}) { + UnknownFieldSet lhs_stack, rhs_stack; + auto& lhs = + lhs_arena ? *Arena::Create(&arena) : lhs_stack; + auto& rhs = + rhs_arena ? *Arena::Create(&arena) : rhs_stack; + PopulateUFS(lhs); + lhs.Swap(&rhs); + } + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithClear) { + Arena arena; + auto* ufs = Arena::Create(&arena); + PopulateUFS(*ufs); + // Clear should not try to delete memory from the arena. + ufs->Clear(); +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksDelete) { + Arena arena; + + auto* ufs = Arena::Create(&arena); + PopulateUFS(*ufs); + + while (ufs->field_count() != 0) { + ufs->DeleteByNumber(ufs->field(0).number()); + } + + ufs = Arena::Create(&arena); + PopulateUFS(*ufs); + ufs->DeleteSubrange(0, ufs->field_count()); +} + TEST_F(UnknownFieldSetTest, SerializeFastAndSlowAreEquivalent) { int size = WireFormat::ComputeUnknownFieldsSize(empty_message_.unknown_fields()); @@ -516,13 +604,15 @@ TEST_F(UnknownFieldSetTest, UnknownEnumValue) { TEST_F(UnknownFieldSetTest, SpaceUsedExcludingSelf) { UnknownFieldSet empty; empty.AddVarint(1, 0); - EXPECT_EQ(sizeof(UnknownField), empty.SpaceUsedExcludingSelf()); + RepeatedField rep; + rep.Add(); + EXPECT_EQ(rep.SpaceUsedExcludingSelf(), empty.SpaceUsedExcludingSelf()); } TEST_F(UnknownFieldSetTest, SpaceUsed) { // Keep shadow vectors to avoid making assumptions about its capacity growth. // We imitate the push back calls here to determine the expected capacity. - std::vector shadow_vector, shadow_vector_group; + RepeatedField shadow_vector, shadow_vector_group; unittest::TestEmptyMessage empty_message; // Make sure an unknown field set has zero space used until a field is @@ -532,8 +622,8 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { UnknownFieldSet* group = nullptr; const auto total = [&] { size_t result = base; - result += shadow_vector.capacity() * sizeof(UnknownField); - result += shadow_vector_group.capacity() * sizeof(UnknownField); + result += shadow_vector.SpaceUsedExcludingSelfLong(); + result += shadow_vector_group.SpaceUsedExcludingSelfLong(); if (str != nullptr) { result += sizeof(std::string); static const size_t sso_capacity = std::string().capacity(); @@ -550,26 +640,26 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { // Make sure each thing we add to the set increases the SpaceUsedLong(). unknown_fields->AddVarint(1, 0); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var"; str = unknown_fields->AddLengthDelimited(1); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str"; str->assign(sizeof(std::string) + 1, 'x'); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str2"; group = unknown_fields->AddGroup(1); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Group"; group->AddVarint(1, 0); - shadow_vector_group.emplace_back(); + shadow_vector_group.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Group2"; unknown_fields->AddVarint(1, 0); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var2"; } From 84af72713d3d5aebfbc79d9763efa82f50692650 Mon Sep 17 00:00:00 2001 From: Chris Kennelly Date: Mon, 22 Jul 2024 08:23:56 -0700 Subject: [PATCH 042/107] Remove Arena::CreateMessage. It has equivalent performance to Arena::Create. PiperOrigin-RevId: 654760266 --- src/google/protobuf/arena.h | 5 +++-- src/google/protobuf/port_def.inc | 4 ++++ src/google/protobuf/port_undef.inc | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index a3cd7b7654..0b3208a180 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -173,8 +173,8 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { inline ~Arena() = default; - // Deprecated. Use Create instead. TODO: depreate OSS version - // once internal migration to Arena::Create is done. +#ifndef PROTOBUF_FUTURE_CREATE_MESSAGE + // Deprecated. Use Create instead. template ABSL_DEPRECATED("Use Create") static T* CreateMessage(Arena* arena, Args&&... args) { @@ -184,6 +184,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { "CreateMessage can only construct types that are ArenaConstructable"); return Create(arena, std::forward(args)...); } +#endif // !PROTOBUF_FUTURE_CREATE_MESSAGE // Allocates an object type T if the arena passed in is not nullptr; // otherwise, returns a heap-allocated object. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 52fa1b315b..a46c7b018f 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -143,6 +143,10 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), // Owner: shaod@, gberg@ #define PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL 1 +// Removes Arena::CreateMessage, as Arena::Create is equivalent +// Owner: ckennelly@, mkruskal@ +#define PROTOBUF_FUTURE_CREATE_MESSAGE 1 + #endif #ifdef PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 0855b44c7f..1434617257 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -84,6 +84,7 @@ #undef PROTOBUF_FUTURE_BREAKING_CHANGES #undef PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL #undef PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE +#undef PROTOBUF_FUTURE_CREATE_MESSAGE #endif // Restore macros that may have been #undef'd in port_def.inc. From 6d25846f4718c37629d8f6df9a5534690da5c498 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 09:39:02 -0700 Subject: [PATCH 043/107] Add 'specific' traits for common gencode message operations. This also changes MessageView and MessageMut to be subtraits of the ViewProxy and MutProxy (which also requires them to have lifetimes now, and a weird extra bounds on ViewProxy) PiperOrigin-RevId: 654788207 --- conformance/conformance_rust.rs | 52 +++++-------- rust/codegen_traits.rs | 80 +++++++++++++++++++- rust/shared.rs | 7 +- src/google/protobuf/compiler/rust/message.cc | 44 +++++++++-- 4 files changed, 141 insertions(+), 42 deletions(-) diff --git a/conformance/conformance_rust.rs b/conformance/conformance_rust.rs index adfe8578ab..bbb0710970 100644 --- a/conformance/conformance_rust.rs +++ b/conformance/conformance_rust.rs @@ -13,6 +13,7 @@ use protobuf_cpp as kernel; use protobuf_upb as kernel; use kernel::Optional::{Set, Unset}; +use kernel::{Message, ParseError}; use std::io::{self, ErrorKind, Read, Write}; use test_messages_edition2023_rust_proto::TestAllTypesEdition2023; @@ -73,52 +74,39 @@ fn do_test(req: &ConformanceRequest) -> ConformanceResponse { Set(bytes) => bytes, }; + fn roundtrip(bytes: &[u8]) -> Result, ParseError> { + T::parse(bytes).map(|msg| msg.serialize().unwrap()) + } + let serialized = match message_type.as_bytes() { b"protobuf_test_messages.proto2.TestAllTypesProto2" => { - if let Ok(msg) = TestAllTypesProto2::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.proto3.TestAllTypesProto3" => { - if let Ok(msg) = TestAllTypesProto3::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.editions.TestAllTypesEdition2023" => { - if let Ok(msg) = TestAllTypesEdition2023::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.editions.proto2.TestAllTypesProto2" => { - if let Ok(msg) = EditionsTestAllTypesProto2::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.editions.proto3.TestAllTypesProto3" => { - if let Ok(msg) = EditionsTestAllTypesProto3::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } _ => panic!("unexpected msg type {message_type}"), }; - resp.set_protobuf_payload(serialized); - return resp; + match serialized { + Ok(serialized) => { + resp.set_protobuf_payload(serialized); + } + Err(_) => { + resp.set_parse_error("failed to parse bytes"); + } + } + + resp } fn main() { diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index 89c2db871d..fe5de9916b 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -7,11 +7,83 @@ //! Traits that are implemeted by codegen types. -use crate::Proxied; +use crate::{MutProxied, MutProxy, ViewProxy}; +use create::Parse; +use read::Serialize; use std::fmt::Debug; +use write::ClearAndParse; -pub trait Message: Default + Debug + Proxied + Send + Sync {} +/// A trait that all generated owned message types implement. +pub trait Message: MutProxied + // Create traits: + + create::Parse + Default + // Read traits: + + Debug + Serialize + // Write traits: + // TODO: Msg should impl Clear. + + ClearAndParse + // Thread safety: + + Send + Sync + // Copy/Clone: + + Clone + {} -pub trait MessageView: Debug + Send + Sync {} +/// A trait that all generated message views implement. +pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> + // Read traits: + + Debug + Serialize + // Thread safety: + + Send + Sync + // Copy/Clone: + + Copy + Clone + { + #[doc(hidden)] + type Message: Message; + } -pub trait MessageMut: Debug + Sync {} +/// A trait that all generated message muts implement. +pub trait MessageMut<'msg>: + MutProxy<'msg, Proxied = Self::Message> + // Read traits: + + Debug + Serialize + // Write traits: + // TODO: MsgMut should impl Clear and ClearAndParse. + // Thread safety: + + Sync + // Copy/Clone: (Neither) +{ + #[doc(hidden)] + type Message: Message; +} + +/// Operations related to constructing a message. Only owned messages implement +/// these traits. +pub(crate) mod create { + + pub trait Parse: Sized { + fn parse(serialized: &[u8]) -> Result; + } +} + +/// Operations related to reading some aspect of a message (methods that would +/// have a `&self` receiver on an owned message). Owned messages, views, and +/// muts all implement these traits. +pub(crate) mod read { + pub trait Serialize { + fn serialize(&self) -> Result, crate::SerializeError>; + } +} + +/// Operations related to mutating a message (methods that would have a `&mut +/// self` receiver on an owned message). Owned messages and muts implement these +/// traits. +pub(crate) mod write { + + pub trait Clear { + fn clear(&mut self); + } + + pub trait ClearAndParse { + fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), crate::ParseError>; + } +} diff --git a/rust/shared.rs b/rust/shared.rs index ec31d65a01..113292c875 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -22,7 +22,12 @@ use std::fmt; /// These are the items protobuf users can access directly. #[doc(hidden)] pub mod __public { - pub use crate::codegen_traits::{Message, MessageMut, MessageView}; + pub use crate::codegen_traits::{ + create::Parse, + read::Serialize, + write::{Clear, ClearAndParse}, + Message, MessageMut, MessageView, + }; pub use crate::r#enum::{Enum, UnknownEnumValue}; pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; pub use crate::optional::Optional; diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 20217a1ae5..d18ece8348 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -953,15 +953,33 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { impl $pb$::Message for $Msg$ {} + impl std::default::Default for $Msg$ { + fn default() -> Self { + Self::new() + } + } + + impl $pb$::Parse for $Msg$ { + fn parse(serialized: &[u8]) -> Result { + Self::parse(serialized) + } + } + impl std::fmt::Debug for $Msg$ { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { $Msg::debug$ } } - impl std::default::Default for $Msg$ { - fn default() -> Self { - Self::new() + impl $pb$::Serialize for $Msg$ { + fn serialize(&self) -> Result, $pb$::SerializeError> { + self.serialize() + } + } + + impl $pb$::ClearAndParse for $Msg$ { + fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { + self.clear_and_parse(data) } } @@ -990,7 +1008,9 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { _phantom: $Phantom$<&'msg ()>, } - impl $pb$::MessageView for $Msg$View<'_> {} + impl<'msg> $pb$::MessageView<'msg> for $Msg$View<'msg> { + type Message = $Msg$; + } impl std::fmt::Debug for $Msg$View<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -998,6 +1018,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::Serialize for $Msg$View<'_> { + fn serialize(&self) -> Result, $pb$::SerializeError> { + self.serialize() + } + } + #[allow(dead_code)] impl<'msg> $Msg$View<'msg> { #[doc(hidden)] @@ -1053,7 +1079,9 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { inner: $pbr$::MutatorMessageRef<'msg>, } - impl $pb$::MessageMut for $Msg$Mut<'_> {} + impl<'msg> $pb$::MessageMut<'msg> for $Msg$Mut<'msg> { + type Message = $Msg$; + } impl std::fmt::Debug for $Msg$Mut<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -1061,6 +1089,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::Serialize for $Msg$Mut<'_> { + fn serialize(&self) -> Result, $pb$::SerializeError> { + self.serialize() + } + } + #[allow(dead_code)] impl<'msg> $Msg$Mut<'msg> { #[doc(hidden)] From d29246f3907ea98a3f2baf991c007ed5b4e46980 Mon Sep 17 00:00:00 2001 From: Tony Liao Date: Mon, 22 Jul 2024 09:41:36 -0700 Subject: [PATCH 044/107] Refactor implicit field presence code generation in GenerateByteSize. ShouldEmitIfNonDefaultCheck is mostly called in MayEmitIfNonDefaultCheck, except in one location in GenerateByteSize. Making MayEmitIfNonDefaultCheck the only caller of ShouldEmitIfNonDefaultCheck seems like a good application of DRY and reduces the possibility of unintended divergence in the future. It also makes future changes to serialization logic easier. There should be no changes to the resulting generated code. PiperOrigin-RevId: 654789393 --- src/google/protobuf/compiler/cpp/message.cc | 51 +++++++++------------ 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 2bd41be347..9a3a3ba80a 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -227,16 +227,16 @@ bool MayEmitIfNonDefaultCheck(io::Printer* p, const std::string& prefix, ABSL_CHECK(!HasHasbit(field)); if (!ShouldEmitNonDefaultCheck(field)) return false; - // SUBTLE: format_string must be a raw string without newline. + // SUBTLE: |format| must be a raw string without newline. // io::Printer::Emit treats the format string as a "raw string" if it doesn't // contain multiple lines. Note that a string of the form "\n($condition$)\n" // (i.e. newline characters are present; there is only one non-empty line) - // will be treated as a multi-line string. + // will still be treated as a multi-line string. // // io::Printer::Emit will print a newline if the input is a multi-line string. - absl::string_view format_string = "if ($condition$) "; + // In this case, we prefer to let the caller handle if-statement braces. p->Emit({{"condition", [&] { EmitNonDefaultCheck(p, prefix, field); }}}, - format_string); + /*format=*/"if ($condition$)"); return true; } @@ -3960,7 +3960,7 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) { // merged only if non-zero (numeric) or non-empty (string). bool emitted_check = MayEmitIfNonDefaultCheck(p, "from.", field); if (emitted_check) { - p->Emit("{\n"); + p->Emit(" {\n"); p->Indent(); } generator.GenerateMergingCode(p); @@ -4236,7 +4236,7 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* p, } else if (field->is_optional()) { bool emitted_check = MayEmitIfNonDefaultCheck(p, "this_.", field); if (emitted_check) { - p->Emit("{\n"); + p->Emit(" {\n"); p->Indent(); } emit_body(); @@ -4778,30 +4778,23 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) { }}, {"check_if_field_present", [&] { - if (HasHasbit(field)) { - if (field->options().weak()) { - p->Emit("if (has_$name$())"); - return; - } - - int has_bit_index = - has_bit_indices_[field->index()]; - p->Emit({{"mask", - absl::StrFormat( - "0x%08xu", - 1u << (has_bit_index % 32))}}, - "if (cached_has_bits & $mask$)"); - } else if (ShouldEmitNonDefaultCheck(field)) { - // Without field presence: field is - // serialized only if it has a non-default - // value. - p->Emit({{"non_default_check", - [&] { - EmitNonDefaultCheck(p, "this_.", - field); - }}}, - "if ($non_default_check$)"); + if (!HasHasbit(field)) { + MayEmitIfNonDefaultCheck(p, "this_.", field); + return; + } + + if (field->options().weak()) { + p->Emit("if (has_$name$())"); + return; } + + int has_bit_index = + has_bit_indices_[field->index()]; + p->Emit( + {{"mask", absl::StrFormat( + "0x%08xu", + 1u << (has_bit_index % 32))}}, + "if (cached_has_bits & $mask$)"); }}}, R"cc( $comment$; From 6290dcd2443a0db65dcdba4dee41e27ad7dd143e Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Mon, 22 Jul 2024 10:39:18 -0700 Subject: [PATCH 045/107] Move elements instead of copying in RepeatedField::ExtractSubrange. Motivation: this matters when Element is Cord. PiperOrigin-RevId: 654813293 --- src/google/protobuf/repeated_field.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index c34746cbd0..5ff41a4c15 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -766,15 +766,17 @@ void RepeatedField::ExtractSubrange(int start, int num, ABSL_DCHECK_GE(num, 0); const int old_size = size(); ABSL_DCHECK_LE(start + num, old_size); + Element* elem = unsafe_elements(); // Save the values of the removed elements if requested. if (elements != nullptr) { - for (int i = 0; i < num; ++i) elements[i] = Get(i + start); + for (int i = 0; i < num; ++i) elements[i] = std::move(elem[i + start]); } // Slide remaining elements down to fill the gap. if (num > 0) { - for (int i = start + num; i < old_size; ++i) Set(i - num, Get(i)); + for (int i = start + num; i < old_size; ++i) + elem[i - num] = std::move(elem[i]); Truncate(old_size - num); } } From 6ebdc7a4d48e74db1174e565ac932e4d4cb8ca87 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Mon, 22 Jul 2024 11:08:16 -0700 Subject: [PATCH 046/107] [ObjC] Expose a helper for stream error. Also mark the helper as not inline able to avoid the code being over duplicated within the file with some compiler optimizations turned on. PiperOrigin-RevId: 654826102 --- objectivec/GPBCodedInputStream.m | 29 +++++++++++-------- .../GPBCodedInputStream_PackagePrivate.h | 1 + 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m index c39648b588..b725600717 100644 --- a/objectivec/GPBCodedInputStream.m +++ b/objectivec/GPBCodedInputStream.m @@ -13,6 +13,10 @@ #import "GPBUtilities_PackagePrivate.h" #import "GPBWireFormat.h" +// TODO: Consider using on other functions to reduce bloat when +// some compiler optimizations are enabled. +#define GPB_NOINLINE __attribute__((noinline)) + NSString *const GPBCodedInputStreamException = GPBNSStringifySymbol(GPBCodedInputStreamException); NSString *const GPBCodedInputStreamUnderlyingErrorKey = @@ -28,7 +32,8 @@ NSString *const GPBCodedInputStreamErrorDomain = // int CodedInputStream::default_recursion_limit_ = 100; static const NSUInteger kDefaultRecursionLimit = 100; -static void RaiseException(NSInteger code, NSString *reason) { +GPB_NOINLINE +void GPBRaiseStreamError(NSInteger code, NSString *reason) { NSDictionary *errorInfo = nil; if ([reason length]) { errorInfo = @{GPBErrorReasonKey : reason}; @@ -44,7 +49,7 @@ static void RaiseException(NSInteger code, NSString *reason) { GPB_INLINE void CheckRecursionLimit(GPBCodedInputStreamState *state) { if (state->recursionDepth >= kDefaultRecursionLimit) { - RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } } @@ -56,19 +61,19 @@ GPB_INLINE void CheckFieldSize(uint64_t size) { if (size > 0x7fffffff) { // TODO: Maybe a different error code for this, but adding one is a breaking // change so reuse an existing one. - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidSize, nil); } } static void CheckSize(GPBCodedInputStreamState *state, size_t size) { size_t newSize = state->bufferPos + size; if (newSize > state->bufferSize) { - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidSize, nil); } if (newSize > state->currentLimit) { // Fast forward to end of currentLimit; state->bufferPos = state->currentLimit; - RaiseException(GPBCodedInputStreamErrorSubsectionLimitReached, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorSubsectionLimitReached, nil); } } @@ -110,7 +115,7 @@ static int64_t ReadRawVarint64(GPBCodedInputStreamState *state) { } shift += 7; } - RaiseException(GPBCodedInputStreamErrorInvalidVarInt, @"Invalid VarInt64"); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidVarInt, @"Invalid VarInt64"); return 0; } @@ -201,12 +206,12 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { state->lastTag = ReadRawVarint32(state); // Tags have to include a valid wireformat. if (!GPBWireFormatIsValidTag(state->lastTag)) { - RaiseException(GPBCodedInputStreamErrorInvalidTag, @"Invalid wireformat in tag."); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Invalid wireformat in tag."); } // Zero is not a valid field number. if (GPBWireFormatGetTagFieldNumber(state->lastTag) == 0) { - RaiseException(GPBCodedInputStreamErrorInvalidTag, - @"A zero field number on the wire is invalid."); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, + @"A zero field number on the wire is invalid."); } return state->lastTag; } @@ -231,7 +236,7 @@ NSString *GPBCodedInputStreamReadRetainedString(GPBCodedInputStreamState *state) NSLog(@"UTF-8 failure, is some field type 'string' when it should be " @"'bytes'?"); #endif - RaiseException(GPBCodedInputStreamErrorInvalidUTF8, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidUTF8, nil); } } return result; @@ -266,7 +271,7 @@ size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byte byteLimit += state->bufferPos; size_t oldLimit = state->currentLimit; if (byteLimit > oldLimit) { - RaiseException(GPBCodedInputStreamErrorInvalidSubsectionLimit, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidSubsectionLimit, nil); } state->currentLimit = byteLimit; return oldLimit; @@ -286,7 +291,7 @@ BOOL GPBCodedInputStreamIsAtEnd(GPBCodedInputStreamState *state) { void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, int32_t value) { if (state->lastTag != value) { - RaiseException(GPBCodedInputStreamErrorInvalidTag, @"Unexpected tag read"); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unexpected tag read"); } } diff --git a/objectivec/GPBCodedInputStream_PackagePrivate.h b/objectivec/GPBCodedInputStream_PackagePrivate.h index fbeecf130c..0c4c3c22a1 100644 --- a/objectivec/GPBCodedInputStream_PackagePrivate.h +++ b/objectivec/GPBCodedInputStream_PackagePrivate.h @@ -53,6 +53,7 @@ typedef struct GPBCodedInputStreamState { CF_EXTERN_C_BEGIN +void GPBRaiseStreamError(NSInteger code, NSString *reason); int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state); double GPBCodedInputStreamReadDouble(GPBCodedInputStreamState *state); From 25ef4467ac425ea9fc59eb41c16e84fb15f67ada Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Mon, 22 Jul 2024 11:18:49 -0700 Subject: [PATCH 047/107] [ObjC] Stop using `xcpretty`. Something has changed between Xcode and xcpretty such that we don't get full logs when something fails to know what file/line is actually failing; so stop using it. PiperOrigin-RevId: 654830446 --- .github/workflows/test_objectivec.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index 1db6c10c88..ea2726dd62 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -57,8 +57,7 @@ jobs: -scheme ProtocolBuffers \ -configuration ${{ matrix.xc_config }} \ -destination "${{ matrix.destination }}" \ - test \ - | xcpretty + test - name: Report ccache stats shell: bash From 9d2dafe1e5aff738cef00aa0ee25550752c31076 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 11:30:36 -0700 Subject: [PATCH 048/107] Add #[doc(hidden)] and a _private arg on SerializedData PiperOrigin-RevId: 654834732 --- rust/cpp.rs | 3 ++- rust/upb.rs | 1 + src/google/protobuf/compiler/rust/message.cc | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/cpp.rs b/rust/cpp.rs index f852dfdfbc..63f79f3f59 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -188,6 +188,7 @@ impl From<&ProtoStr> for PtrAndLen { /// This struct is ABI-compatible with the equivalent struct on the C++ side. It /// owns (and drops) its data. #[repr(C)] +#[doc(hidden)] pub struct SerializedData { /// Owns the memory. data: NonNull, @@ -195,7 +196,7 @@ pub struct SerializedData { } impl SerializedData { - pub fn new() -> Self { + pub fn new(_private: Private) -> Self { Self { data: NonNull::dangling(), len: 0 } } diff --git a/rust/upb.rs b/rust/upb.rs index 2c0084da36..82798965dd 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -58,6 +58,7 @@ impl ScratchSpace { } } +#[doc(hidden)] pub type SerializedData = upb::OwnedArenaBox<[u8]>; impl IntoProxied for SerializedData { diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index d18ece8348..d8b354d6ac 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -61,7 +61,7 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: ctx.Emit({{"serialize_thunk", ThunkName(ctx, msg, "serialize")}}, R"rs( - let mut serialized_data = $pbr$::SerializedData::new(); + let mut serialized_data = $pbr$::SerializedData::new($pbi$::Private); let success = unsafe { $serialize_thunk$(self.raw_msg(), &mut serialized_data) }; From 9e38dce6165d5e3f6ce402d4ff687bfa3d88bcb0 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Mon, 22 Jul 2024 11:32:26 -0700 Subject: [PATCH 049/107] fold Ptr into hpb namespace PiperOrigin-RevId: 654835437 --- hpb/hpb.h | 34 +++++++++-------- hpb/repeated_field.h | 12 +++--- hpb_generator/gen_messages.cc | 24 ++++++------ hpb_generator/gen_repeated_fields.cc | 12 +++--- hpb_generator/names.cc | 5 ++- hpb_generator/tests/test_generated.cc | 55 +++++++++++++-------------- 6 files changed, 72 insertions(+), 70 deletions(-) diff --git a/hpb/hpb.h b/hpb/hpb.h index 9e18057638..171da7896e 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -24,11 +24,6 @@ namespace hpb { using Arena = ::upb::Arena; -} - -namespace protos { -using hpb::Arena; -class ExtensionRegistry; template using Proxy = std::conditional_t::value, @@ -87,6 +82,13 @@ class Ptr final { template Ptr(T* m) -> Ptr; +} // namespace hpb + +namespace protos { +using hpb::Arena; +using hpb::Ptr; +class ExtensionRegistry; + inline absl::string_view UpbStrToStringView(upb_StringView str) { return absl::string_view(str.data, str.size); } @@ -101,7 +103,7 @@ inline upb_StringView UpbStrFromStringView(absl::string_view str, } template -typename T::Proxy CreateMessage(::protos::Arena& arena) { +typename T::Proxy CreateMessage(::hpb::Arena& arena) { return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), arena.ptr()); } @@ -310,19 +312,19 @@ typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { template void DeepCopy(Ptr source_message, T* target_message) { static_assert(!std::is_const_v); - DeepCopy(source_message, protos::Ptr(target_message)); + DeepCopy(source_message, Ptr(target_message)); } template void DeepCopy(const T* source_message, Ptr target_message) { static_assert(!std::is_const_v); - DeepCopy(protos::Ptr(source_message), target_message); + DeepCopy(Ptr(source_message), target_message); } template void DeepCopy(const T* source_message, T* target_message) { static_assert(!std::is_const_v); - DeepCopy(protos::Ptr(source_message), protos::Ptr(target_message)); + DeepCopy(Ptr(source_message), Ptr(target_message)); } template @@ -371,7 +373,7 @@ template & id) { - return HasExtension(protos::Ptr(message), id); + return HasExtension(Ptr(message), id); } template & id) { - ClearExtension(::protos::Ptr(message), id); + ClearExtension(Ptr(message), id); } template & id, const Extension& value) { - return ::protos::SetExtension(::protos::Ptr(message), id, value); + return ::protos::SetExtension(Ptr(message), id, value); } template & id, Extension&& value) { - return ::protos::SetExtension(::protos::Ptr(message), id, + return ::protos::SetExtension(Ptr(message), id, std::forward(value)); } @@ -459,7 +461,7 @@ template & id, Ptr value) { - return ::protos::SetExtension(::protos::Ptr(message), id, value); + return ::protos::SetExtension(Ptr(message), id, value); } template > GetExtension( const T* message, const ::protos::internal::ExtensionIdentifier& id) { - return GetExtension(protos::Ptr(message), id); + return GetExtension(Ptr(message), id); } template @@ -522,7 +524,7 @@ ABSL_MUST_USE_RESULT bool Parse( T* message, absl::string_view bytes, const ::protos::ExtensionRegistry& extension_registry) { static_assert(!std::is_const_v); - return Parse(protos::Ptr(message, bytes, extension_registry)); + return Parse(Ptr(message, bytes, extension_registry)); } template diff --git a/hpb/repeated_field.h b/hpb/repeated_field.h index 3162f6a9d7..396dc3a3ff 100644 --- a/hpb/repeated_field.h +++ b/hpb/repeated_field.h @@ -94,7 +94,7 @@ class RepeatedFieldProxy : RepeatedFieldProxyBase(arr, arena) {} RepeatedFieldProxy(upb_Array* arr, upb_Arena* arena) : RepeatedFieldProxyMutableBase(arr, arena) {} - // Constructor used by ::protos::Ptr. + // Constructor used by ::hpb::Ptr. RepeatedFieldProxy(const RepeatedFieldProxy&) = default; // T::CProxy [] operator specialization. @@ -147,7 +147,7 @@ class RepeatedFieldProxy reverse_iterator rend() const { return reverse_iterator(begin()); } private: - friend class ::protos::Ptr; + friend class ::hpb::Ptr; }; // RepeatedField proxy for repeated strings. @@ -175,7 +175,7 @@ class RepeatedFieldStringProxy // Mutable constructor. RepeatedFieldStringProxy(upb_Array* arr, upb_Arena* arena) : RepeatedFieldProxyMutableBase(arr, arena) {} - // Constructor used by ::protos::Ptr. + // Constructor used by ::hpb::Ptr. RepeatedFieldStringProxy(const RepeatedFieldStringProxy&) = default; reference operator[](size_t n) const { return begin()[n]; } @@ -222,7 +222,7 @@ class RepeatedFieldScalarProxy : RepeatedFieldProxyBase(arr, arena) {} RepeatedFieldScalarProxy(upb_Array* arr, upb_Arena* arena) : RepeatedFieldProxyMutableBase(arr, arena) {} - // Constructor used by ::protos::Ptr. + // Constructor used by ::hpb::Ptr. RepeatedFieldScalarProxy(const RepeatedFieldScalarProxy&) = default; T operator[](size_t n) const { @@ -285,10 +285,10 @@ class RepeatedField { // We would like to reference T::CProxy. Validate forwarding header design. using ValueProxy = std::conditional_t< kIsScalar, T, - std::conditional_t>>; + std::conditional_t>>; using ValueCProxy = std::conditional_t< kIsScalar, const T, - std::conditional_t>>; + std::conditional_t>>; using Access = std::conditional_t< kIsScalar, internal::RepeatedFieldScalarProxy, std::conditional_t, diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index d2576743c4..ed0ff87100 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -237,7 +237,7 @@ void WriteModelPublicDeclaration( arena_ = owned_arena_.ptr(); upb_Arena_Fuse(arena_, arena); } - ::protos::Arena owned_arena_; + ::hpb::Arena owned_arena_; friend struct ::protos::internal::PrivateAccess; friend Proxy; friend CProxy; @@ -248,7 +248,7 @@ void WriteModelPublicDeclaration( const ::protos::ExtensionRegistry& extension_registry, int options)); friend upb_Arena* ::protos::internal::GetArena<$0>($0* message); - friend upb_Arena* ::protos::internal::GetArena<$0>(::protos::Ptr<$0> message); + friend upb_Arena* ::protos::internal::GetArena<$0>(::hpb::Ptr<$0> message); friend $0(::hpb::internal::MoveMessage<$0>(upb_Message* msg, upb_Arena* arena)); )cc", ClassName(descriptor), MessageName(descriptor), @@ -293,22 +293,22 @@ void WriteModelProxyDeclaration(const protobuf::Descriptor* descriptor, $0Proxy(upb_Message* msg, upb_Arena* arena) : internal::$0Access(($1*)msg, arena) {} - friend $0::Proxy(::protos::CreateMessage<$0>(::protos::Arena& arena)); + friend $0::Proxy(::protos::CreateMessage<$0>(::hpb::Arena& arena)); friend $0::Proxy(::protos::internal::CreateMessageProxy<$0>( upb_Message*, upb_Arena*)); friend struct ::protos::internal::PrivateAccess; friend class RepeatedFieldProxy; friend class $0CProxy; friend class $0Access; - friend class ::protos::Ptr<$0>; - friend class ::protos::Ptr; + friend class ::hpb::Ptr<$0>; + friend class ::hpb::Ptr; static const upb_MiniTable* minitable() { return $0::minitable(); } friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0Proxy>( const $0Proxy* message); friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0Proxy>( - ::protos::Ptr<$0Proxy> message); + ::hpb::Ptr<$0Proxy> message); friend upb_Arena* ::protos::internal::GetArena<$2>($2* message); - friend upb_Arena* ::protos::internal::GetArena<$2>(::protos::Ptr<$2> message); + friend upb_Arena* ::protos::internal::GetArena<$2>(::hpb::Ptr<$2> message); static void Rebind($0Proxy& lhs, const $0Proxy& rhs) { lhs.msg_ = rhs.msg_; lhs.arena_ = rhs.arena_; @@ -349,13 +349,13 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, : internal::$0Access(($1*)msg, arena){}; friend struct ::protos::internal::PrivateAccess; friend class RepeatedFieldProxy; - friend class ::protos::Ptr<$0>; - friend class ::protos::Ptr; + friend class ::hpb::Ptr<$0>; + friend class ::hpb::Ptr; static const upb_MiniTable* minitable() { return $0::minitable(); } friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0CProxy>( const $0CProxy* message); friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0CProxy>( - ::protos::Ptr<$0CProxy> message); + ::hpb::Ptr<$0CProxy> message); static void Rebind($0CProxy& lhs, const $0CProxy& rhs) { lhs.msg_ = rhs.msg_; @@ -369,7 +369,7 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, void WriteDefaultInstanceHeader(const protobuf::Descriptor* message, Output& output) { - output(" static ::protos::Ptr default_instance();\n", + output(" static ::hpb::Ptr default_instance();\n", ClassName(message)); } @@ -445,7 +445,7 @@ void WriteMessageImplementation( output( R"cc( - ::protos::Ptr $0::default_instance() { + ::hpb::Ptr $0::default_instance() { return ::protos::internal::CreateMessage<$0>( (upb_Message *)_$0_default_instance_.msg, _$0_default_instance_.arena); diff --git a/hpb_generator/gen_repeated_fields.cc b/hpb_generator/gen_repeated_fields.cc index f010294d43..163648e3a8 100644 --- a/hpb_generator/gen_repeated_fields.cc +++ b/hpb_generator/gen_repeated_fields.cc @@ -87,7 +87,7 @@ void WriteRepeatedFieldsInMessageHeader(const protobuf::Descriptor* desc, R"cc( $1 $2(size_t index) const; const ::protos::RepeatedField::CProxy $2() const; - ::protos::Ptr<::protos::RepeatedField<$4>> mutable_$2(); + ::hpb::Ptr<::protos::RepeatedField<$4>> mutable_$2(); absl::StatusOr<$0> add_$2(); $0 mutable_$2(size_t index) const; )cc", @@ -102,7 +102,7 @@ void WriteRepeatedFieldsInMessageHeader(const protobuf::Descriptor* desc, R"cc( $0 $1(size_t index) const; const ::protos::RepeatedField<$0>::CProxy $1() const; - ::protos::Ptr<::protos::RepeatedField<$0>> mutable_$1(); + ::hpb::Ptr<::protos::RepeatedField<$0>> mutable_$1(); bool add_$1($0 val); void set_$1(size_t index, $0 val); bool resize_$1(size_t len); @@ -113,7 +113,7 @@ void WriteRepeatedFieldsInMessageHeader(const protobuf::Descriptor* desc, R"cc( $0 $1(size_t index) const; const ::protos::RepeatedField<$0>::CProxy $1() const; - ::protos::Ptr<::protos::RepeatedField<$0>> mutable_$1(); + ::hpb::Ptr<::protos::RepeatedField<$0>> mutable_$1(); bool add_$1($0 val); void set_$1(size_t index, $0 val); bool resize_$1(size_t len); @@ -177,7 +177,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, const upb_Array* arr = _$3_$4_$5(msg_, &size); return ::protos::RepeatedField::CProxy(arr, arena_); }; - ::protos::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { + ::hpb::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { size_t size; upb_Array* arr = _$3_$4_$6(msg_, &size, arena_); return ::protos::RepeatedField<$1>::Proxy(arr, arena_); @@ -243,7 +243,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, const upb_Array* arr = _$3_$4_$5(msg_, &size); return ::protos::RepeatedField<$1>::CProxy(arr, arena_); }; - ::protos::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { + ::hpb::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { size_t size; upb_Array* arr = _$3_$4_$6(msg_, &size, arena_); return ::protos::RepeatedField<$1>::Proxy(arr, arena_); @@ -307,7 +307,7 @@ void WriteRepeatedScalarAccessor(const protobuf::Descriptor* message, const upb_Array* arr = _$3_$4_$5(msg_, &size); return ::protos::RepeatedField<$1>::CProxy(arr, arena_); }; - ::protos::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { + ::hpb::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { size_t size; upb_Array* arr = _$3_$4_$6(msg_, &size, arena_); return ::protos::RepeatedField<$1>::Proxy(arr, arena_); diff --git a/hpb_generator/names.cc b/hpb_generator/names.cc index 5ea3da9d6a..864eca5aaf 100644 --- a/hpb_generator/names.cc +++ b/hpb_generator/names.cc @@ -17,6 +17,7 @@ namespace protobuf = ::proto2; namespace { +// TODO: b/346865271 append ::hpb instead of ::protos after namespace swap std::string NamespaceFromPackageName(absl::string_view package_name) { return absl::StrCat(absl::StrReplaceAll(package_name, {{".", "::"}}), "::protos"); @@ -82,7 +83,7 @@ std::string ClassName(const protobuf::Descriptor* descriptor) { const protobuf::Descriptor* parent = descriptor->containing_type(); std::string res; // Classes in global namespace without package names are prefixed - // by protos_ to avoid collision with C compiler structs defined in + // by hpb_ to avoid collision with C compiler structs defined in // proto.upb.h. if ((parent && parent->file()->package().empty()) || descriptor->file()->package().empty()) { @@ -155,7 +156,7 @@ std::string MessagePtrConstType(const protobuf::FieldDescriptor* field, bool is_const) { ABSL_DCHECK(field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE); std::string maybe_const = is_const ? "const " : ""; - return "::protos::Ptr<" + maybe_const + + return "::hpb::Ptr<" + maybe_const + QualifiedClassName(field->message_type()) + ">"; } diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index 578f2d8b8d..eb6ca07300 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -700,7 +700,7 @@ TEST(CppGeneratedCode, ClearExtensionWithEmptyExtension) { TEST(CppGeneratedCode, ClearExtensionWithEmptyExtensionPtr) { TestModel model; - ::protos::Ptr recursive_child = model.mutable_recursive_child(); + ::hpb::Ptr recursive_child = model.mutable_recursive_child(); ::protos::ClearExtension(recursive_child, theme); EXPECT_EQ(false, ::protos::HasExtension(recursive_child, theme)); } @@ -726,13 +726,12 @@ TEST(CppGeneratedCode, SetExtension) { TEST(CppGeneratedCode, SetExtensionWithPtr) { ::hpb::Arena arena_model; - ::protos::Ptr model = - ::protos::CreateMessage(arena_model); + ::hpb::Ptr model = ::protos::CreateMessage(arena_model); void* prior_message; { // Use a nested scope to make sure the arenas are fused correctly. ::hpb::Arena arena; - ::protos::Ptr extension1 = + ::hpb::Ptr extension1 = ::protos::CreateMessage(arena); extension1->set_ext_name("Hello World"); prior_message = ::protos::internal::GetInternalMsg(extension1); @@ -749,7 +748,7 @@ TEST(CppGeneratedCode, SetExtensionWithPtr) { #ifndef _MSC_VER TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { ::hpb::Arena arena; - ::protos::Ptr model = ::protos::CreateMessage(arena); + ::hpb::Ptr model = ::protos::CreateMessage(arena); ThemeExtension extension1; ContainerExtension extension2; @@ -770,11 +769,11 @@ TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { ::hpb::Arena arena; - ::protos::Ptr model = ::protos::CreateMessage(arena); + ::hpb::Ptr model = ::protos::CreateMessage(arena); void* prior_message; { // Use a nested scope to make sure the arenas are fused correctly. - ::protos::Ptr extension1 = + ::hpb::Ptr extension1 = ::protos::CreateMessage(arena); extension1->set_ext_name("Hello World"); prior_message = ::protos::internal::GetInternalMsg(extension1); @@ -791,9 +790,9 @@ TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { TEST(CppGeneratedCode, SetExtensionFusingFailureShouldCopy) { // Use an initial block to disallow fusing. char initial_block[1000]; - protos::Arena arena(initial_block, sizeof(initial_block)); + hpb::Arena arena(initial_block, sizeof(initial_block)); - protos::Ptr model = protos::CreateMessage(arena); + hpb::Ptr model = ::protos::CreateMessage(arena); ThemeExtension extension1; extension1.set_ext_name("Hello World"); @@ -861,7 +860,7 @@ TEST(CppGeneratedCode, GetExtensionOnMutableChild) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - ::protos::Ptr mutable_recursive_child = + ::hpb::Ptr mutable_recursive_child = model.mutable_recursive_child(); EXPECT_EQ(false, ::protos::HasExtension(mutable_recursive_child, theme)); EXPECT_EQ( @@ -877,13 +876,13 @@ TEST(CppGeneratedCode, GetExtensionOnImmutableChild) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - ::protos::Ptr mutable_recursive_child = + ::hpb::Ptr mutable_recursive_child = model.mutable_recursive_child(); EXPECT_EQ(false, ::protos::HasExtension(mutable_recursive_child, theme)); EXPECT_EQ( true, ::protos::SetExtension(mutable_recursive_child, theme, extension1).ok()); - ::protos::Ptr recursive_child = model.recursive_child(); + ::hpb::Ptr recursive_child = model.recursive_child(); EXPECT_EQ("Hello World", ::protos::GetExtension(recursive_child, theme).value()->ext_name()); } @@ -915,7 +914,7 @@ TEST(CppGeneratedCode, SerializeNestedMessageUsingArena) { TestModel model; model.mutable_recursive_child()->set_str1("Hello World"); ::upb::Arena arena; - ::protos::Ptr child = model.recursive_child(); + ::hpb::Ptr child = model.recursive_child(); absl::StatusOr bytes = ::protos::Serialize(child, arena); EXPECT_EQ(true, bytes.ok()); TestModel parsed_model = ::protos::Parse(bytes.value()).value(); @@ -945,7 +944,7 @@ TEST(CppGeneratedCode, ParseIntoPtrToModel) { ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); - ::protos::Ptr parsed_model = + ::hpb::Ptr parsed_model = ::protos::CreateMessage(arena); EXPECT_TRUE(::protos::Parse(parsed_model, bytes.value())); EXPECT_EQ("Test123", parsed_model->str1()); @@ -992,15 +991,15 @@ TEST(CppGeneratedCode, NameCollisions) { TEST(CppGeneratedCode, SharedPointer) { std::shared_ptr model = std::make_shared(); ::upb::Arena arena; - auto bytes = protos::Serialize(model.get(), arena); - EXPECT_TRUE(protos::Parse(model.get(), bytes.value())); + auto bytes = ::protos::Serialize(model.get(), arena); + EXPECT_TRUE(::protos::Parse(model.get(), bytes.value())); } TEST(CppGeneratedCode, UniquePointer) { auto model = std::make_unique(); ::upb::Arena arena; - auto bytes = protos::Serialize(model.get(), arena); - EXPECT_TRUE(protos::Parse(model.get(), bytes.value())); + auto bytes = ::protos::Serialize(model.get(), arena); + EXPECT_TRUE(::protos::Parse(model.get(), bytes.value())); } TEST(CppGeneratedCode, Assignment) { @@ -1039,21 +1038,21 @@ TEST(CppGeneratedCode, PtrConstructor) { TEST(CppGeneratedCode, MutableToProxy) { TestModel model; - ::protos::Ptr child = model.mutable_child_model_1(); + ::hpb::Ptr child = model.mutable_child_model_1(); (void)child; } TEST(CppGeneratedCode, ProxyToCProxy) { TestModel model; - ::protos::Ptr child = model.mutable_child_model_1(); - ::protos::Ptr child2 = child; + ::hpb::Ptr child = model.mutable_child_model_1(); + ::hpb::Ptr child2 = child; (void)child2; } TEST(CppGeneratedCode, MutableAccessorsAreHiddenInCProxy) { TestModel model; - ::protos::Ptr proxy = &model; - ::protos::Ptr cproxy = proxy; + ::hpb::Ptr proxy = &model; + ::hpb::Ptr cproxy = proxy; const auto test_const_accessors = [](auto p) { // We don't want to run it, just check it compiles. @@ -1105,7 +1104,7 @@ TEST(CppGeneratedCode, MutableAccessorsAreHiddenInCProxy) { test_mutable_accessors(cproxy, false); } -bool ProxyToCProxyMethod(::protos::Ptr child) { +bool ProxyToCProxyMethod(::hpb::Ptr child) { return child->child_str1() == "text in child"; } @@ -1118,7 +1117,7 @@ TEST(CppGeneratedCode, PassProxyToCProxy) { TEST(CppGeneratedCode, PtrImplicitConversion) { TestModel model; model.set_int64(5); - ::protos::Ptr model_ptr = &model; + ::hpb::Ptr model_ptr = &model; EXPECT_EQ(model_ptr->int64(), 5); } @@ -1163,7 +1162,7 @@ TEST(CppGeneratedCode, CanInvokeClearMessageWithPtr) { model.set_int64(5); auto new_child = model.add_child_models(); // Clear using Ptr - auto ptr = ::protos::Ptr(&model); + auto ptr = ::hpb::Ptr(&model); ::protos::ClearMessage(ptr); // Successful clear EXPECT_FALSE(model.has_int64()); @@ -1186,8 +1185,8 @@ bool CanCallClearMessage() { } TEST(CppGeneratedCode, CannotInvokeClearMessageWithConstPtr) { - EXPECT_TRUE(CanCallClearMessage<::protos::Ptr>()); - EXPECT_FALSE(CanCallClearMessage<::protos::Ptr>()); + EXPECT_TRUE(CanCallClearMessage<::hpb::Ptr>()); + EXPECT_FALSE(CanCallClearMessage<::hpb::Ptr>()); } TEST(CppGeneratedCode, CannotInvokeClearMessageWithConstRawPtr) { From a1c53e4da26000121e26734959889cece7365be8 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 11:35:04 -0700 Subject: [PATCH 050/107] Remove +?Sized bounds which simply aren't honored anyway. When you have a `T: SomeTrait + ?Sized` and `trait SomeTrait:Sized`, the ?Sized has no effect (its not able to "remove" the requirement). PiperOrigin-RevId: 654836513 --- rust/cpp.rs | 4 +-- rust/map.rs | 71 +++++++++++++++++++++---------------------------- rust/proxied.rs | 2 +- rust/upb.rs | 6 ++--- 4 files changed, 37 insertions(+), 46 deletions(-) diff --git a/rust/cpp.rs b/rust/cpp.rs index 63f79f3f59..7d4cf56d63 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -691,8 +691,8 @@ impl UntypedMapIterator { from_ffi_value: impl FnOnce(FfiValue) -> View<'a, V>, ) -> Option<(View<'a, K>, View<'a, V>)> where - K: Proxied + ?Sized + 'a, - V: ProxiedInMapValue + ?Sized + 'a, + K: Proxied + 'a, + V: ProxiedInMapValue + 'a, { if self.at_end() { return None; diff --git a/rust/map.rs b/rust/map.rs index 170f50c97d..0e7953f872 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -59,19 +59,19 @@ impl<'msg, K: ?Sized, V: ?Sized> std::fmt::Debug for MapMut<'msg, K, V> { } } -pub struct Map> { +pub struct Map> { inner: InnerMap, _phantom: PhantomData<(PhantomData, PhantomData)>, } // SAFETY: `Map` is Sync because it does not implement interior mutability. -unsafe impl> Sync for Map {} +unsafe impl> Sync for Map {} // SAFETY: `Map` is Send because it's not bound to a specific thread e.g. // it does not use thread-local data or similar. -unsafe impl> Send for Map {} +unsafe impl> Send for Map {} -impl> Drop for Map { +impl> Drop for Map { fn drop(&mut self) { // SAFETY: // - `drop` is only called once. @@ -82,7 +82,7 @@ impl> Drop for Map { pub trait ProxiedInMapValue: Proxied where - K: Proxied + ?Sized, + K: Proxied, { fn map_new(_private: Private) -> Map; @@ -104,17 +104,15 @@ where fn map_iter_next<'a>(iter: &mut MapIter<'a, K, Self>) -> Option<(View<'a, K>, View<'a, Self>)>; } -impl + ?Sized> Proxied for Map { +impl> Proxied for Map { type View<'msg> = MapView<'msg, K, V> where K: 'msg, V: 'msg; } -impl + ?Sized> MutProxied for Map { +impl> MutProxied for Map { type Mut<'msg> = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> Proxy<'msg> - for MapView<'msg, K, V> -{ +impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, V> { type Proxied = Map; fn as_view(&self) -> View<'_, Self::Proxied> { @@ -129,14 +127,9 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> Proxy<'msg> } } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg> - for MapView<'msg, K, V> -{ -} +impl<'msg, K: Proxied, V: ProxiedInMapValue> ViewProxy<'msg> for MapView<'msg, K, V> {} -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> Proxy<'msg> - for MapMut<'msg, K, V> -{ +impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapMut<'msg, K, V> { type Proxied = Map; fn as_view(&self) -> View<'_, Self::Proxied> { @@ -151,9 +144,7 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> Proxy<'msg> } } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> MutProxy<'msg> - for MapMut<'msg, K, V> -{ +impl<'msg, K: Proxied, V: ProxiedInMapValue> MutProxy<'msg> for MapMut<'msg, K, V> { fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { MapMut { inner: self.inner, _phantom: PhantomData } } @@ -168,8 +159,8 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> MutProxy<'msg> impl Map where - K: Proxied + ?Sized, - V: ProxiedInMapValue + ?Sized, + K: Proxied, + V: ProxiedInMapValue, { pub fn new() -> Self { V::map_new(Private) @@ -195,8 +186,8 @@ where impl Default for Map where - K: Proxied + ?Sized, - V: ProxiedInMapValue + ?Sized, + K: Proxied, + V: ProxiedInMapValue, { fn default() -> Self { Map::new() @@ -220,8 +211,8 @@ impl<'msg, K: ?Sized, V: ?Sized> MapView<'msg, K, V> { impl<'msg, K, V> MapView<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { pub fn get<'a>(self, key: impl Into>) -> Option> where @@ -278,8 +269,8 @@ impl<'msg, K: ?Sized, V: ?Sized> MapMut<'msg, K, V> { impl<'msg, K, V> MapMut<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { pub fn len(&self) -> usize { self.as_view().len() @@ -378,8 +369,8 @@ impl<'msg, K: ?Sized, V: ?Sized> MapIter<'msg, K, V> { impl<'msg, K, V> Iterator for MapIter<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type Item = (View<'msg, K>, View<'msg, V>); @@ -390,8 +381,8 @@ where impl<'msg, K, V> IntoIterator for MapView<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'msg, K, V>; type Item = (View<'msg, K>, View<'msg, V>); @@ -403,8 +394,8 @@ where impl<'msg, K, V> IntoIterator for &'msg Map where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'msg, K, V>; type Item = (View<'msg, K>, View<'msg, V>); @@ -417,8 +408,8 @@ where impl<'a, 'msg, K, V> IntoIterator for &'a MapView<'msg, K, V> where 'msg: 'a, - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'msg, K, V>; type Item = (View<'msg, K>, View<'msg, V>); @@ -431,8 +422,8 @@ where impl<'a, 'msg, K, V> IntoIterator for &'a MapMut<'msg, K, V> where 'msg: 'a, - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'a, K, V>; // The View's are valid for 'a instead of 'msg. @@ -446,8 +437,8 @@ where impl<'msg, 'k, 'v, KView, VView, K, V> Extend<(KView, VView)> for MapMut<'msg, K, V> where - K: Proxied + ?Sized + 'msg + 'k, - V: ProxiedInMapValue + ?Sized + 'msg + 'v, + K: Proxied + 'msg + 'k, + V: ProxiedInMapValue + 'msg + 'v, KView: Into>, VView: IntoProxied, { diff --git a/rust/proxied.rs b/rust/proxied.rs index 5606824441..f252b2e879 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -96,7 +96,7 @@ pub type Mut<'msg, T> = ::Mut<'msg>; /// This trait is intentionally made non-object-safe to prevent a potential /// future incompatible change. pub trait Proxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { - type Proxied: 'msg + Proxied + ?Sized; + type Proxied: 'msg + Proxied; /// Converts a borrow into a `View` with the lifetime of that borrow. /// diff --git a/rust/upb.rs b/rust/upb.rs index 82798965dd..0af32ba382 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -464,7 +464,7 @@ pub fn free_enum_repeated( } /// Returns a static empty RepeatedView. -pub fn empty_array() -> RepeatedView<'static, T> { +pub fn empty_array() -> RepeatedView<'static, T> { // TODO: Consider creating a static empty array in C. // Use `i32` for a shared empty repeated for all repeated types in the program. @@ -484,8 +484,8 @@ pub fn empty_array() -> RepeatedView<'static, T> /// Returns a static empty MapView. pub fn empty_map() -> MapView<'static, K, V> where - K: Proxied + ?Sized, - V: ProxiedInMapValue + ?Sized, + K: Proxied, + V: ProxiedInMapValue, { // TODO: Consider creating a static empty map in C. From c37dd57a7076961be11f9e0695d4f5c97a980d6f Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Mon, 22 Jul 2024 11:36:28 -0700 Subject: [PATCH 051/107] Remove deprecated GetPrototype usages in google/protobuf/ to git rid of the warinings PiperOrigin-RevId: 654836977 --- python/google/protobuf/internal/descriptor_pool_test.py | 2 +- python/google/protobuf/internal/message_factory_test.py | 2 +- python/google/protobuf/internal/python_message.py | 9 ++++++--- python/google/protobuf/internal/symbol_database_test.py | 5 ----- python/google/protobuf/pyext/cpp_message.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index e9ddae6d16..1eed7af25b 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -525,7 +525,7 @@ class DescriptorPoolTestBase(object): unittest_import_pb2.DESCRIPTOR.serialized_pb)) pool.Add(descriptor_pb2.FileDescriptorProto.FromString( unittest_pb2.DESCRIPTOR.serialized_pb)) - message_class = message_factory.MessageFactory(pool).GetPrototype( + message_class = message_factory.GetMessageClass( pool.FindMessageTypeByName( unittest_pb2.TestAllTypes.DESCRIPTOR.full_name)) _CheckDefaultValues(message_class()) diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index e72ecde094..09467cf72d 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -65,7 +65,7 @@ class MessageFactoryTest(unittest.TestCase): result = cls.FromString(reserialized) self.assertEqual(msg, result) - def testGetPrototype(self): + def testGetMessageClass(self): db = descriptor_database.DescriptorDatabase() pool = descriptor_pool.DescriptorPool(db) db.Add(self.factory_test1_fd) diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index fabc6aa078..28f599f759 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -72,7 +72,7 @@ class GeneratedProtocolMessageType(type): mydescriptor = Descriptor(.....) factory = symbol_database.Default() factory.pool.AddDescriptor(mydescriptor) - MyProtoClass = factory.GetPrototype(mydescriptor) + MyProtoClass = message_factory.GetMessageClass(mydescriptor) myproto_instance = MyProtoClass() myproto.foo_field = 23 ... @@ -120,7 +120,7 @@ class GeneratedProtocolMessageType(type): # to achieve similar results. # # This most commonly happens in `text_format.py` when using descriptors from - # a custom pool; it calls symbol_database.Global().getPrototype() on a + # a custom pool; it calls message_factory.GetMessageClass() on a # descriptor which already has an existing concrete class. new_class = getattr(descriptor, '_concrete_class', None) if new_class: @@ -988,7 +988,10 @@ def _InternalUnpackAny(msg): if descriptor is None: return None - message_class = factory.GetPrototype(descriptor) + # Unable to import message_factory at top becaue of circular import. + # pylint: disable=g-import-not-at-top + from google.protobuf import message_factory + message_class = message_factory.GetMessageClass(descriptor) message = message_class() message.ParseFromString(msg.value) diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index 47675fcf3d..45cc979667 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -35,11 +35,6 @@ class SymbolDatabaseTest(unittest.TestCase): db.RegisterServiceDescriptor(unittest_pb2._TESTSERVICE) return db - def testGetPrototype(self): - instance = self._Database().GetPrototype( - unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertTrue(instance is unittest_pb2.TestAllTypes) - def testGetMessages(self): messages = self._Database().GetMessages( ['google/protobuf/unittest.proto']) diff --git a/python/google/protobuf/pyext/cpp_message.py b/python/google/protobuf/pyext/cpp_message.py index 623b52fbff..54cfe30646 100644 --- a/python/google/protobuf/pyext/cpp_message.py +++ b/python/google/protobuf/pyext/cpp_message.py @@ -34,7 +34,7 @@ class GeneratedProtocolMessageType(_message.MessageMeta): mydescriptor = Descriptor(.....) factory = symbol_database.Default() factory.pool.AddDescriptor(mydescriptor) - MyProtoClass = factory.GetPrototype(mydescriptor) + MyProtoClass = message_factory.GetMessageClass(mydescriptor) myproto_instance = MyProtoClass() myproto.foo_field = 23 ... From 972e5499eb88aa2ad08a394cfaea4d380454c68c Mon Sep 17 00:00:00 2001 From: Chris Kennelly Date: Mon, 22 Jul 2024 12:08:24 -0700 Subject: [PATCH 052/107] Automated rollback of commit 84af72713d3d5aebfbc79d9763efa82f50692650. PiperOrigin-RevId: 654848026 --- src/google/protobuf/arena.h | 5 ++--- src/google/protobuf/port_def.inc | 4 ---- src/google/protobuf/port_undef.inc | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 0b3208a180..a3cd7b7654 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -173,8 +173,8 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { inline ~Arena() = default; -#ifndef PROTOBUF_FUTURE_CREATE_MESSAGE - // Deprecated. Use Create instead. + // Deprecated. Use Create instead. TODO: depreate OSS version + // once internal migration to Arena::Create is done. template ABSL_DEPRECATED("Use Create") static T* CreateMessage(Arena* arena, Args&&... args) { @@ -184,7 +184,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { "CreateMessage can only construct types that are ArenaConstructable"); return Create(arena, std::forward(args)...); } -#endif // !PROTOBUF_FUTURE_CREATE_MESSAGE // Allocates an object type T if the arena passed in is not nullptr; // otherwise, returns a heap-allocated object. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index a46c7b018f..52fa1b315b 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -143,10 +143,6 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), // Owner: shaod@, gberg@ #define PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL 1 -// Removes Arena::CreateMessage, as Arena::Create is equivalent -// Owner: ckennelly@, mkruskal@ -#define PROTOBUF_FUTURE_CREATE_MESSAGE 1 - #endif #ifdef PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 1434617257..0855b44c7f 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -84,7 +84,6 @@ #undef PROTOBUF_FUTURE_BREAKING_CHANGES #undef PROTOBUF_FUTURE_DESCRIPTOR_EXTENSION_DECL #undef PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE -#undef PROTOBUF_FUTURE_CREATE_MESSAGE #endif // Restore macros that may have been #undef'd in port_def.inc. From af794965f9e8eef398f26fe4a47cd06c26cb018a Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Mon, 22 Jul 2024 12:42:06 -0700 Subject: [PATCH 053/107] [ObjC] Add test around enum unknown values. PiperOrigin-RevId: 654858735 --- objectivec/Tests/GPBMessageTests.m | 52 +++++++- objectivec/Tests/unittest_objc.proto | 187 +++++++++++++++------------ 2 files changed, 152 insertions(+), 87 deletions(-) diff --git a/objectivec/Tests/GPBMessageTests.m b/objectivec/Tests/GPBMessageTests.m index d598a48b60..e70f898e3e 100644 --- a/objectivec/Tests/GPBMessageTests.m +++ b/objectivec/Tests/GPBMessageTests.m @@ -5,14 +5,14 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#import "GPBTestUtilities.h" - #import +#import "GPBArray.h" #import "GPBArray_PackagePrivate.h" #import "GPBDescriptor.h" #import "GPBDictionary_PackagePrivate.h" #import "GPBMessage_PackagePrivate.h" +#import "GPBTestUtilities.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" #import "objectivec/Tests/Unittest.pbobjc.h" @@ -2022,6 +2022,54 @@ XCTAssertEqual([msgPrime.mumbleArray valueAtIndex:4], EnumTestMsg_MyEnum_NegTwo); } +- (void)testCloseEnumsValuesOutOfRange { + // The unknown values should all make it into the unknown fields. + EnumTestMsg *msg1 = [EnumTestMsg message]; + msg1.bar = EnumTestMsg_MyEnum_NegTwo; + msg1.baz = EnumTestMsg_MyEnum_Two; + [msg1.mumbleArray addValue:EnumTestMsg_MyEnum_Two]; + [msg1.mumbleArray addValue:EnumTestMsg_MyEnum_NegTwo]; + + NSData *data = [msg1 data]; + XCTAssertNotNil(data); + + EnumTestMsgPrime *msg2 = [EnumTestMsgPrime parseFromData:data error:NULL]; + XCTAssertNotNil(msg2); + XCTAssertEqualObjects(data, [msg2 data]); + XCTAssertFalse(msg2.hasBar); + XCTAssertFalse(msg2.hasBaz); + XCTAssertEqual(msg2.mumbleArray_Count, 0U); + + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] initFromMessage:msg2] autorelease]; + XCTAssertEqual(ufs.count, 4U); + uint64_t varint; + XCTAssertTrue([ufs getFirst:EnumTestMsg_FieldNumber_Bar varint:&varint]); + XCTAssertEqual(varint, (uint64_t)EnumTestMsg_MyEnum_NegTwo); + XCTAssertTrue([ufs getFirst:EnumTestMsg_FieldNumber_Baz varint:&varint]); + XCTAssertEqual(varint, (uint64_t)EnumTestMsg_MyEnum_Two); + NSArray *fields = [ufs fields:EnumTestMsg_FieldNumber_MumbleArray]; + XCTAssertEqual(fields.count, 2U); + XCTAssertEqual(fields[0].varint, (uint64_t)EnumTestMsg_MyEnum_Two); + XCTAssertEqual(fields[1].varint, (uint64_t)EnumTestMsg_MyEnum_NegTwo); + + GPBUnknownFieldSet *unknownFields = msg2.unknownFields; + XCTAssertNotNil(unknownFields); + XCTAssertEqual(unknownFields.countOfFields, 3U); + XCTAssertTrue([unknownFields hasField:EnumTestMsg_FieldNumber_Bar]); + XCTAssertTrue([unknownFields hasField:EnumTestMsg_FieldNumber_Baz]); + XCTAssertTrue([unknownFields hasField:EnumTestMsg_FieldNumber_MumbleArray]); + GPBUnknownField *field = [unknownFields getField:EnumTestMsg_FieldNumber_Bar]; + XCTAssertEqual(field.varintList.count, 1U); + XCTAssertEqual([field.varintList valueAtIndex:0], (uint64_t)EnumTestMsg_MyEnum_NegTwo); + field = [unknownFields getField:EnumTestMsg_FieldNumber_Baz]; + XCTAssertEqual(field.varintList.count, 1U); + XCTAssertEqual([field.varintList valueAtIndex:0], (uint64_t)EnumTestMsg_MyEnum_Two); + field = [unknownFields getField:EnumTestMsg_FieldNumber_MumbleArray]; + XCTAssertEqual(field.varintList.count, 2U); + XCTAssertEqual([field.varintList valueAtIndex:0], (uint64_t)EnumTestMsg_MyEnum_Two); + XCTAssertEqual([field.varintList valueAtIndex:1], (uint64_t)EnumTestMsg_MyEnum_NegTwo); +} + - (void)testReservedWordNaming { // names.cc has some special handing to make sure that some "reserved" objc // names get renamed in a way so they don't conflict. diff --git a/objectivec/Tests/unittest_objc.proto b/objectivec/Tests/unittest_objc.proto index 4f0e348569..b7b67fc372 100644 --- a/objectivec/Tests/unittest_objc.proto +++ b/objectivec/Tests/unittest_objc.proto @@ -7,11 +7,11 @@ syntax = "proto2"; +package objc.protobuf.tests; + import "google/protobuf/any.proto"; import "objectivec/Tests/unittest.proto"; -package objc.protobuf.tests; - // Explicit empty prefix, tests some validations code paths also. option objc_class_prefix = ""; @@ -78,143 +78,142 @@ message self { } enum autorelease { - retain = 1; - release = 2; + retain = 1; + release = 2; retainCount = 3; } // Singular // Objective C Keywords - optional bool id = 1; - optional bool _cmd = 2; + optional bool id = 1; + optional bool _cmd = 2; // super is used as submessage above - optional bool in = 4; - optional bool out = 5; - optional bool inout = 6; - optional bool bycopy = 7; - optional bool byref = 8; - optional bool oneway = 9; - optional bool self = 10; - optional bool instancetype = 11; - optional bool nullable = 12; - optional bool nonnull = 13; - optional bool nil = 14; + optional bool in = 4; + optional bool out = 5; + optional bool inout = 6; + optional bool bycopy = 7; + optional bool byref = 8; + optional bool oneway = 9; + optional bool self = 10; + optional bool instancetype = 11; + optional bool nullable = 12; + optional bool nonnull = 13; + optional bool nil = 14; // Nil and nil can't be in the same message - optional bool YES = 16; - optional bool NO = 17; - optional bool weak = 18; + optional bool YES = 16; + optional bool NO = 17; + optional bool weak = 18; // Some C/C++ Keywords - optional bool case = 30; - optional bool if = 31; - optional bool and_eq = 32; - optional bool public = 33; - optional bool private = 34; - optional bool typename = 35; - optional bool static_cast = 36; - optional bool typeof = 37; - optional bool restrict = 38; - optional bool NULL = 39; + optional bool case = 30; + optional bool if = 31; + optional bool and_eq = 32; + optional bool public = 33; + optional bool private = 34; + optional bool typename = 35; + optional bool static_cast = 36; + optional bool typeof = 37; + optional bool restrict = 38; + optional bool NULL = 39; // Some NSObject Methods - optional bool dealloc = 110; - optional bool isProxy = 111; - optional bool copy = 112; - optional bool description = 113; - optional bool zone = 114; - optional bool className = 115; - optional bool __retain_OA = 116; - optional bool CAMLType = 117; - optional bool isNSDictionary__ = 118; + optional bool dealloc = 110; + optional bool isProxy = 111; + optional bool copy = 112; + optional bool description = 113; + optional bool zone = 114; + optional bool className = 115; + optional bool __retain_OA = 116; + optional bool CAMLType = 117; + optional bool isNSDictionary__ = 118; optional bool accessibilityLabel = 119; // Some Objc "keywords" that we shouldn't // have to worry about because they // can only appear in specialized areas. - optional bool assign = 200; - optional bool getter = 201; - optional bool setter = 202; - optional bool atomic = 203; - optional bool nonatomic = 204; - optional bool strong = 205; - optional bool null_resettable = 206; - optional bool readonly = 207; + optional bool assign = 200; + optional bool getter = 201; + optional bool setter = 202; + optional bool atomic = 203; + optional bool nonatomic = 204; + optional bool strong = 205; + optional bool null_resettable = 206; + optional bool readonly = 207; // Some GPBMessage methods - optional bool clear = 300; - optional bool data = 301; - optional bool descriptor = 302; - optional bool delimitedData = 303; + optional bool clear = 300; + optional bool data = 301; + optional bool descriptor = 302; + optional bool delimitedData = 303; // Some MacTypes - optional bool Fixed = 400; - optional bool Point = 401; - optional bool FixedPoint = 402; - optional bool Style = 403; + optional bool Fixed = 400; + optional bool Point = 401; + optional bool FixedPoint = 402; + optional bool Style = 403; // C/C++ reserved identifiers - optional bool _Generic = 500; - optional bool __block = 501; + optional bool _Generic = 500; + optional bool __block = 501; // Try a keyword as a type - optional autorelease SubEnum = 1000; + optional autorelease SubEnum = 1000; optional group New = 2000 { - optional string copy = 1; + optional string copy = 1; } optional group MutableCopy = 2001 { optional int32 extensionRegistry = 1; } extensions 3000 to 3999; - } enum retain { - count = 4; - initialized = 5; + count = 4; + initialized = 5; serializedSize = 6; } message ObjCPropertyNaming { // Test that the properties properly get things all caps. - optional string url = 1; + optional string url = 1; optional string thumbnail_url = 2; - optional string url_foo = 3; + optional string url_foo = 3; optional string some_url_blah = 4; - optional string http = 5; - optional string https = 6; + optional string http = 5; + optional string https = 6; // This one doesn't. - repeated string urls = 7; + repeated string urls = 7; } // EnumValueShortName: The short names shouldn't get suffixes/prefixes. enum Foo { SERIALIZED_SIZE = 1; - SIZE = 2; - OTHER = 3; + SIZE = 2; + OTHER = 3; } // EnumValueShortName: The enum name gets a prefix. enum Category { - RED = 1; + RED = 1; BLUE = 2; } // EnumValueShortName: Twist case, full name gets PB, but the short names // should still end up correct. enum Time { - BASE = 1; - RECORD = 2; - SOMETHING_ELSE = 3; + BASE = 1; + RECORD = 2; + SOMETHING_ELSE = 3; } extend self { - repeated int32 debugDescription = 3000 [packed = true]; - repeated int64 finalize = 3001 [packed = true]; - repeated uint32 hash = 3002 [packed = true]; - repeated uint64 classForCoder = 3003 [packed = true]; - repeated sint32 byref = 3004 [packed = true]; + repeated int32 debugDescription = 3000 [packed = true]; + repeated int64 finalize = 3001 [packed = true]; + repeated uint32 hash = 3002 [packed = true]; + repeated uint64 classForCoder = 3003 [packed = true]; + repeated sint32 byref = 3004 [packed = true]; } // Test handing of fields that start with init*. @@ -700,13 +699,17 @@ message JustToScopeExtensions { repeated string mutableCopy_val_lower_complex_repeated = 2711; repeated string mutableCopy_Val_upper_complex_repeated = 2712; - repeated string mutableCopyvalue_lower_no_underscore_complex_repeated = 2713; - repeated string mutableCopyValue_upper_no_underscore_complex_repeated = 2714; + repeated string mutableCopyvalue_lower_no_underscore_complex_repeated = + 2713; + repeated string mutableCopyValue_upper_no_underscore_complex_repeated = + 2714; repeated int32 mutableCopy_val_lower_primitive_repeated = 2715; repeated int32 mutableCopy_Val_upper_primitive_repeated = 2716; - repeated int32 mutableCopyvalue_lower_no_underscore_primitive_repeated = 2717; - repeated int32 mutableCopyValue_upper_no_underscore_primitive_repeated = 2718; + repeated int32 mutableCopyvalue_lower_no_underscore_primitive_repeated = + 2717; + repeated int32 mutableCopyValue_upper_no_underscore_primitive_repeated = + 2718; repeated self mutableCopy_val_lower_message_repeated = 2719; repeated self mutableCopy_Val_upper_message_repeated = 2720; @@ -781,9 +784,9 @@ message ObjcWeirdDefaults { // Used to confirm negative enum values work as expected. message EnumTestMsg { enum MyEnum { - ZERO = 0; - ONE = 1; - TWO = 2; + ZERO = 0; + ONE = 1; + TWO = 2; NEG_ONE = -1; NEG_TWO = -2; } @@ -794,6 +797,20 @@ message EnumTestMsg { repeated MyEnum mumble = 4; } +message EnumTestMsgPrime { + enum MyEnumPrime { + ZERO = 0; + ONE = 1; + NEG_ONE = -1; + // Lacks 2, -2. + } + optional MyEnumPrime foo = 1; + optional MyEnumPrime bar = 2 [default = ONE]; + optional MyEnumPrime baz = 3 [default = NEG_ONE]; + + repeated MyEnumPrime mumble = 4; +} + // Test case for https://github.com/protocolbuffers/protobuf/issues/1453 // Message with no explicit defaults, but a non zero default for an enum. message MessageWithOneBasedEnum { From d44c0d577c937552ef43a12ea8a175dbea8d7a69 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Mon, 22 Jul 2024 13:29:17 -0700 Subject: [PATCH 054/107] [ObjC] Parsing helper and tests around unknown fields. Make a new internal api for collecting up the unknown group fields that will be used in a future change. Add testing for the new api and for unknown field parsing of groups in general. PiperOrigin-RevId: 654875605 --- objectivec/GPBCodedInputStream.m | 54 ++++++ .../GPBCodedInputStream_PackagePrivate.h | 3 + objectivec/Tests/GPBCodedInputStreamTests.m | 102 +++++++++- objectivec/Tests/GPBUnknownFieldSetTest.m | 175 ++++++++++++++++++ objectivec/Tests/GPBUnknownFieldsTest.m | 160 ++++++++++++++++ 5 files changed, 493 insertions(+), 1 deletion(-) diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m index b725600717..4ee03ef580 100644 --- a/objectivec/GPBCodedInputStream.m +++ b/objectivec/GPBCodedInputStream.m @@ -267,6 +267,60 @@ NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *sta return result; } +static void SkipToEndGroupInternal(GPBCodedInputStreamState *state, uint32_t endGroupTag) { + CheckRecursionLimit(state); + ++state->recursionDepth; + while (YES) { + uint32_t tag = GPBCodedInputStreamReadTag(state); + if (tag == endGroupTag || tag == 0) { + GPBCodedInputStreamCheckLastTagWas(state, endGroupTag); // Will fail for end of input. + --state->recursionDepth; + return; + } + switch (GPBWireFormatGetTagWireType(tag)) { + case GPBWireFormatVarint: + (void)ReadRawVarint64(state); + break; + case GPBWireFormatFixed64: + SkipRawData(state, sizeof(uint64_t)); + break; + case GPBWireFormatLengthDelimited: { + uint64_t size = ReadRawVarint64(state); + CheckFieldSize(size); + size_t size2 = (size_t)size; // Cast safe on 32bit because of CheckFieldSize() above. + SkipRawData(state, size2); + break; + } + case GPBWireFormatStartGroup: + SkipToEndGroupInternal(state, GPBWireFormatMakeTag(GPBWireFormatGetTagFieldNumber(tag), + GPBWireFormatEndGroup)); + break; + case GPBWireFormatEndGroup: + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unmatched end group"); + break; + case GPBWireFormatFixed32: + SkipRawData(state, sizeof(uint32_t)); + break; + } + } +} + +// This doesn't include the start group, but it collects all bytes until the end group including +// the end group tag. +NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state, + int32_t fieldNumber) { + // Better have just read the start of the group. + GPBCodedInputStreamCheckLastTagWas(state, + GPBWireFormatMakeTag(fieldNumber, GPBWireFormatStartGroup)); + const uint8_t *start = state->bytes + state->bufferPos; + SkipToEndGroupInternal(state, GPBWireFormatMakeTag(fieldNumber, GPBWireFormatEndGroup)); + // This will be after the end group tag. + const uint8_t *end = state->bytes + state->bufferPos; + return [[NSData alloc] initWithBytesNoCopy:(void *)start + length:(NSUInteger)(end - start) + freeWhenDone:NO]; +} + size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit) { byteLimit += state->bufferPos; size_t oldLimit = state->currentLimit; diff --git a/objectivec/GPBCodedInputStream_PackagePrivate.h b/objectivec/GPBCodedInputStream_PackagePrivate.h index 0c4c3c22a1..720bd60ee8 100644 --- a/objectivec/GPBCodedInputStream_PackagePrivate.h +++ b/objectivec/GPBCodedInputStream_PackagePrivate.h @@ -76,6 +76,9 @@ NSData *GPBCodedInputStreamReadRetainedBytes(GPBCodedInputStreamState *state) __attribute((ns_returns_retained)); NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *state) __attribute((ns_returns_retained)); +NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state, + int32_t fieldNumber) + __attribute((ns_returns_retained)); size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit); void GPBCodedInputStreamPopLimit(GPBCodedInputStreamState *state, size_t oldLimit); diff --git a/objectivec/Tests/GPBCodedInputStreamTests.m b/objectivec/Tests/GPBCodedInputStreamTests.m index 3dae80d2ac..6bf775dded 100644 --- a/objectivec/Tests/GPBCodedInputStreamTests.m +++ b/objectivec/Tests/GPBCodedInputStreamTests.m @@ -6,12 +6,14 @@ // https://developers.google.com/open-source/licenses/bsd #import -#import "GPBTestUtilities.h" #import "GPBCodedInputStream.h" +#import "GPBCodedInputStream_PackagePrivate.h" #import "GPBCodedOutputStream.h" +#import "GPBTestUtilities.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUtilities_PackagePrivate.h" +#import "GPBWireFormat.h" #import "objectivec/Tests/Unittest.pbobjc.h" @interface CodedInputStreamTests : GPBTestCase @@ -444,4 +446,102 @@ } } +- (void)assertReadByteToEndGroupFails:(NSData*)data { + GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data]; + uint32_t tag = [input readTag]; + XCTAssertThrows(GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy( + &input->state_, GPBWireFormatGetTagFieldNumber(tag))); +} + +- (void)assertReadByteToEndGroup:(NSData*)data value:(NSData*)value { + GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data]; + uint32_t tag = [input readTag]; + NSData* readValue = GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy( + &input->state_, GPBWireFormatGetTagFieldNumber(tag)); + XCTAssertNotNil(readValue); + XCTAssertEqualObjects(readValue, value); + [readValue release]; +} + +static NSData* DataForGroupsOfDepth(NSUInteger depth) { + NSMutableData* data = [NSMutableData dataWithCapacity:0]; + + uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + + byte = 8; // 8 = 0b1000, -> field 1/varint + [data appendBytes:&byte length:1]; + byte = 1; // 1 -> varint value of 1 + [data appendBytes:&byte length:1]; + + byte = 36; // 36 = 0b100100 -> field 4/end group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + return data; +} + +- (void)testBytesToEndGroup { + // 35 = 0b100011 -> field 4/start group + // 36 = 0b100100 -> field 4/end group + // 43 = 0b101011 -> field 5/end group + // 44 = 0b101100 -> field 5/end group + // 8 = 0b1000, 1 -> field 1/varint, value of 1 + // 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678 + // 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64, + // value of 0x123456789abcdef0LL + // 50 = 0b110010, 0x0 -> field 6/length delimited, length 0 + // 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42 + // 0 -> field 0 which is invalid/varint + // 15 = 0b1111 -> field 1, wire type 7 which is invalid + + [self assertReadByteToEndGroup:bytes(35, 36) value:bytes(36)]; // empty group + [self assertReadByteToEndGroup:bytes(35, 8, 1, 36) value:bytes(8, 1, 36)]; // varint + [self assertReadByteToEndGroup:bytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) // fixed32 + value:bytes(21, 0x78, 0x56, 0x34, 0x12, 36)]; + [self assertReadByteToEndGroup:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, + 36) // fixed64 + value:bytes(25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, 36)]; + [self assertReadByteToEndGroup:bytes(35, 50, 0, 36) + value:bytes(50, 0, 36)]; // length delimited, length 0 + [self assertReadByteToEndGroup:bytes(35, 50, 1, 42, 36) + value:bytes(50, 1, 42, 36)]; // length delimited, length 1, byte 42 + [self assertReadByteToEndGroup:bytes(35, 43, 44, 36) value:bytes(43, 44, 36)]; // Sub group + [self assertReadByteToEndGroup:bytes(35, 8, 1, 43, 8, 1, 44, + 36) // varint and sub group with varint + value:bytes(8, 1, 43, 8, 1, 44, 36)]; + + [self assertReadByteToEndGroupFails:bytes(35, 0, 36)]; // Invalid field number + [self assertReadByteToEndGroupFails:bytes(35, 15, 36)]; // Invalid wire type + [self assertReadByteToEndGroupFails:bytes(35, 21, 0x78, 0x56, 0x34)]; // truncated fixed32 + [self assertReadByteToEndGroupFails:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, + 0x34)]; // truncated fixed64 + + // Mising end group + [self assertReadByteToEndGroupFails:bytes(35)]; + [self assertReadByteToEndGroupFails:bytes(35, 8, 1)]; + [self assertReadByteToEndGroupFails:bytes(35, 43)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1)]; + + // Wrong end group + [self assertReadByteToEndGroupFails:bytes(35, 44)]; + [self assertReadByteToEndGroupFails:bytes(35, 8, 1, 44)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 36)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 36)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 44, 44)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 44, 44)]; + + // This is the same limit as within GPBCodedInputStream. + const NSUInteger kDefaultRecursionLimit = 100; + // That depth parses. + NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit); + [self assertReadByteToEndGroup:testData + value:[testData subdataWithRange:NSMakeRange(1, testData.length - 1)]]; + // One more level deep fails. + testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1); + [self assertReadByteToEndGroupFails:testData]; +} + @end diff --git a/objectivec/Tests/GPBUnknownFieldSetTest.m b/objectivec/Tests/GPBUnknownFieldSetTest.m index 10478ac772..1e055b0dd0 100644 --- a/objectivec/Tests/GPBUnknownFieldSetTest.m +++ b/objectivec/Tests/GPBUnknownFieldSetTest.m @@ -6,6 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd #import "GPBTestUtilities.h" +#import "GPBUnknownFieldSet.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" @@ -376,6 +377,180 @@ XCTAssertEqual(0x7FFFFFFFFFFFFFFFULL, [field2.varintList valueAtIndex:0]); } +static NSData* DataForGroupsOfDepth(NSUInteger depth) { + NSMutableData* data = [NSMutableData dataWithCapacity:0]; + + uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + + byte = 8; // 8 = 0b1000, -> field 1/varint + [data appendBytes:&byte length:1]; + byte = 1; // 1 -> varint value of 1 + [data appendBytes:&byte length:1]; + + byte = 36; // 36 = 0b100100 -> field 4/end group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + return data; +} + +- (void)testParsingNestingGroupData { + // 35 = 0b100011 -> field 4/start group + // 36 = 0b100100 -> field 4/end group + // 43 = 0b101011 -> field 5/end group + // 44 = 0b101100 -> field 5/end group + // 8 = 0b1000, 1 -> field 1/varint, value of 1 + // 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678 + // 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64, + // value of 0x123456789abcdef0LL + // 50 = 0b110010, 0x0 -> field 6/length delimited, length 0 + // 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42 + // 0 -> field 0 which is invalid/varint + // 15 = 0b1111 -> field 1, wire type 7 which is invalid + + TestEmptyMessage* m = [TestEmptyMessage parseFromData:DataFromBytes(35, 36) + error:NULL]; // empty group + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + GPBUnknownField* field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + GPBUnknownFieldSet* group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)0); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 36) error:NULL]; // varint + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:1]; + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], 1); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) + error:NULL]; // fixed32 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:2]; + XCTAssertEqual(field.fixed32List.count, (NSUInteger)1); + XCTAssertEqual([field.fixed32List valueAtIndex:0], 0x12345678); + + m = [TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, + 36) + error:NULL]; // fixed64 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:3]; + XCTAssertEqual(field.fixed64List.count, (NSUInteger)1); + XCTAssertEqual([field.fixed64List valueAtIndex:0], 0x123456789abcdef0LL); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36) + error:NULL]; // length delimited, length 0 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:6]; + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1); + XCTAssertEqualObjects(field.lengthDelimitedList[0], [NSData data]); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 1, 42, 36) + error:NULL]; // length delimited, length 1, byte 42 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:6]; + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1); + XCTAssertEqualObjects(field.lengthDelimitedList[0], DataFromBytes(42)); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 36) error:NULL]; // Sub group + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)1); + field = [group getField:5]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)0); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 43, 8, 2, 44, 36) + error:NULL]; // varint and sub group with varint + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)2); + field = [group getField:1]; + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], 1); + field = [group getField:5]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:1]; + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], 2); + + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 0, 36) + error:NULL]); // Invalid field number + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 15, 36) + error:NULL]); // Invalid wire type + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34) + error:NULL]); // truncated fixed32 + XCTAssertNil([TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, + 0x34) + error:NULL]); // truncated fixed64 + + // Mising end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1) error:NULL]); + + // Wrong end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 44, 44) error:NULL]); + + // This is the same limit as within GPBCodedInputStream. + const NSUInteger kDefaultRecursionLimit = 100; + // That depth parses. + NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit); + m = [TestEmptyMessage parseFromData:testData error:NULL]; + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + for (NSUInteger i = 0; i < kDefaultRecursionLimit; ++i) { + XCTAssertEqual(field.varintList.count, (NSUInteger)0); + XCTAssertEqual(field.fixed32List.count, (NSUInteger)0); + XCTAssertEqual(field.fixed64List.count, (NSUInteger)0); + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0); + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)1); + field = [group getField:(i < (kDefaultRecursionLimit - 1) ? 4 : 1)]; + } + // field is of the inner most group + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], (NSUInteger)1); + XCTAssertEqual(field.fixed32List.count, (NSUInteger)0); + XCTAssertEqual(field.fixed64List.count, (NSUInteger)0); + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0); + XCTAssertEqual(field.groupList.count, (NSUInteger)0); + // One more level deep fails. + testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1); + XCTAssertNil([TestEmptyMessage parseFromData:testData error:NULL]); +} + #pragma mark - Field tests // Some tests directly on fields since the dictionary in FieldSet can gate // testing some of these. diff --git a/objectivec/Tests/GPBUnknownFieldsTest.m b/objectivec/Tests/GPBUnknownFieldsTest.m index 9726d8361f..5565f674b5 100644 --- a/objectivec/Tests/GPBUnknownFieldsTest.m +++ b/objectivec/Tests/GPBUnknownFieldsTest.m @@ -847,4 +847,164 @@ XCTAssertEqual(varint, 0x7FFFFFFFFFFFFFFFL); } +static NSData* DataForGroupsOfDepth(NSUInteger depth) { + NSMutableData* data = [NSMutableData dataWithCapacity:0]; + + uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + + byte = 8; // 8 = 0b1000, -> field 1/varint + [data appendBytes:&byte length:1]; + byte = 1; // 1 -> varint value of 1 + [data appendBytes:&byte length:1]; + + byte = 36; // 36 = 0b100100 -> field 4/end group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + return data; +} + +- (void)testParsingNestingGroupData { + // 35 = 0b100011 -> field 4/start group + // 36 = 0b100100 -> field 4/end group + // 43 = 0b101011 -> field 5/end group + // 44 = 0b101100 -> field 5/end group + // 8 = 0b1000, 1 -> field 1/varint, value of 1 + // 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678 + // 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64, + // value of 0x123456789abcdef0LL + // 50 = 0b110010, 0x0 -> field 6/length delimited, length 0 + // 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42 + // 0 -> field 0 which is invalid/varint + // 15 = 0b1111 -> field 1, wire type 7 which is invalid + + TestEmptyMessage* m = [TestEmptyMessage parseFromData:DataFromBytes(35, 36) + error:NULL]; // empty group + GPBUnknownFields* ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + GPBUnknownFields* group = [ufs firstGroup:4]; + XCTAssertTrue(group.empty); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 36) error:NULL]; // varint + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + uint64_t varint = 0; + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 1); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) + error:NULL]; // fixed32 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + uint32_t fixed32 = 0; + XCTAssertTrue([group getFirst:2 fixed32:&fixed32]); + XCTAssertEqual(fixed32, 0x12345678); + + m = [TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, + 36) + error:NULL]; // fixed64 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + uint64_t fixed64 = 0; + XCTAssertTrue([group getFirst:3 fixed64:&fixed64]); + XCTAssertEqual(fixed64, 0x123456789abcdef0LL); + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36) + error:NULL]; // length delimited, length 0 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + NSData* lengthDelimited = [group firstLengthDelimited:6]; + XCTAssertEqualObjects(lengthDelimited, [NSData data]); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 1, 42, 36) + error:NULL]; // length delimited, length 1, byte 42 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + lengthDelimited = [group firstLengthDelimited:6]; + XCTAssertEqualObjects(lengthDelimited, DataFromBytes(42)); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 36) error:NULL]; // Sub group + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + group = [group firstGroup:5]; + XCTAssertTrue(group.empty); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 43, 8, 2, 44, 36) + error:NULL]; // varint and sub group with varint + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)2); + varint = 0; + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 1); + group = [group firstGroup:5]; + XCTAssertEqual(group.count, (NSUInteger)1); + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 2); + + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 0, 36) + error:NULL]); // Invalid field number + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 15, 36) + error:NULL]); // Invalid wire type + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34) + error:NULL]); // truncated fixed32 + XCTAssertNil([TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, + 0x34) + error:NULL]); // truncated fixed64 + + // Mising end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1) error:NULL]); + + // Wrong end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 44, 44) error:NULL]); + + // This is the same limit as within GPBCodedInputStream. + const NSUInteger kDefaultRecursionLimit = 100; + // That depth parses. + NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit); + m = [TestEmptyMessage parseFromData:testData error:NULL]; + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + for (NSUInteger i = 1; i < kDefaultRecursionLimit; ++i) { + XCTAssertEqual(group.count, (NSUInteger)1); + group = [group firstGroup:4]; + } + // group is now the inner most group. + XCTAssertEqual(group.count, (NSUInteger)1); + varint = 0; + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 1); + // One more level deep fails. + testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1); + XCTAssertNil([TestEmptyMessage parseFromData:testData error:NULL]); +} + @end From 9f9cb7a10e5c49c2e81b8352292e4e84a5eb7cfa Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 14:32:10 -0700 Subject: [PATCH 055/107] Move proto_toolchain from rules_proto to protobuf The toolchain type is consumed by proto_library and produced by proto_toolchain rule. As such it's a private dependency, because both rules are now part of protobuf repo. There are some early adopters of --incompatible_enable_proto_toolchain_resolution that might be broken from this: grpc, rules_go, rules_ts, rules_rust, rules_lint, because they have implementation that is not using proto_common. Those repositories need to define their own proto_lang_toolchain and consume it with proto_common.compile. PiperOrigin-RevId: 654897871 --- MODULE.bazel | 5 ++--- bazel/private/BUILD.bazel | 9 +++++++++ bazel/private/proto_toolchain_rule.bzl | 8 ++------ bazel/private/toolchain_helpers.bzl | 2 +- bazel/toolchains/BUILD.bazel | 1 + bazel/toolchains/proto_toolchain.bzl | 3 ++- protobuf_deps.bzl | 11 ----------- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index fcf2eeaa95..05128b355c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -27,9 +27,6 @@ bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "zlib", version = "1.3.1") bazel_dep(name = "bazel_features", version = "1.13.0", repo_name = "proto_bazel_features") -# TODO: remove after toolchain types are moved to protobuf -bazel_dep(name = "rules_proto", version = "4.0.0") - SUPPORTED_PYTHON_VERSIONS = [ "3.8", "3.9", @@ -99,3 +96,5 @@ use_repo(maven, "maven") # Development dependencies bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest", dev_dependency = True) bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True) +# rules_proto are needed for @com_google_protobuf_v25.0 used in //compatibility/... tests +bazel_dep(name = "rules_proto", version = "4.0.0", dev_dependency = True) \ No newline at end of file diff --git a/bazel/private/BUILD.bazel b/bazel/private/BUILD.bazel index 407c325916..c90632d7b0 100644 --- a/bazel/private/BUILD.bazel +++ b/bazel/private/BUILD.bazel @@ -10,6 +10,10 @@ load("//bazel/private:native_bool_flag.bzl", "native_bool_flag") licenses(["notice"]) +toolchain_type( + name = "proto_toolchain_type", +) + bzl_library( name = "upb_proto_library_internal_bzl", srcs = [ @@ -40,6 +44,11 @@ bzl_library( "proto_toolchain_rule.bzl", ], visibility = ["//bazel:__subpackages__"], + deps = [ + ":toolchain_helpers_bzl", + "//bazel/common:proto_common_bzl", + "//bazel/common:proto_lang_toolchain_info_bzl", + ], ) bzl_library( diff --git a/bazel/private/proto_toolchain_rule.bzl b/bazel/private/proto_toolchain_rule.bzl index 9487a06026..8a11fac9db 100644 --- a/bazel/private/proto_toolchain_rule.bzl +++ b/bazel/private/proto_toolchain_rule.bzl @@ -1,13 +1,9 @@ """A Starlark implementation of the proto_toolchain rule.""" -load("//bazel/common:proto_common.bzl", "proto_common") load("//bazel/common:proto_lang_toolchain_info.bzl", "ProtoLangToolchainInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") def _impl(ctx): - kwargs = {} - if getattr(proto_common, "INCOMPATIBLE_PASS_TOOLCHAIN_TYPE", False): - kwargs["toolchain_type"] = "@rules_proto//proto:toolchain_type" - return [ DefaultInfo( files = depset(), @@ -23,7 +19,7 @@ def _impl(ctx): protoc_opts = ctx.fragments.proto.experimental_protoc_opts, progress_message = ctx.attr.progress_message, mnemonic = ctx.attr.mnemonic, - **kwargs + toolchain_type = toolchains.PROTO_TOOLCHAIN, ), ), ] diff --git a/bazel/private/toolchain_helpers.bzl b/bazel/private/toolchain_helpers.bzl index e58c9d6a0e..aa49eb8cf4 100644 --- a/bazel/private/toolchain_helpers.bzl +++ b/bazel/private/toolchain_helpers.bzl @@ -45,5 +45,5 @@ toolchains = struct( find_toolchain = _find_toolchain, if_legacy_toolchain = _if_legacy_toolchain, INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION = _incompatible_toolchain_resolution, - PROTO_TOOLCHAIN = "@rules_proto//proto:toolchain_type", + PROTO_TOOLCHAIN = "//bazel/private:proto_toolchain_type", ) diff --git a/bazel/toolchains/BUILD.bazel b/bazel/toolchains/BUILD.bazel index 34a45d5bbd..2c21426b15 100644 --- a/bazel/toolchains/BUILD.bazel +++ b/bazel/toolchains/BUILD.bazel @@ -8,6 +8,7 @@ bzl_library( visibility = ["//visibility:public"], deps = [ "//bazel/private:proto_toolchain_rule_bzl", + "//bazel/private:toolchain_helpers_bzl", ], ) diff --git a/bazel/toolchains/proto_toolchain.bzl b/bazel/toolchains/proto_toolchain.bzl index 171e08cad5..1bad37b3a6 100644 --- a/bazel/toolchains/proto_toolchain.bzl +++ b/bazel/toolchains/proto_toolchain.bzl @@ -4,6 +4,7 @@ The macro additionally creates toolchain target when toolchain_type is given. """ load("//bazel/private:proto_toolchain_rule.bzl", _proto_toolchain_rule = "proto_toolchain") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") def proto_toolchain(*, name, proto_compiler, exec_compatible_with = []): """Creates a proto_toolchain and toolchain target for proto_library. @@ -19,7 +20,7 @@ def proto_toolchain(*, name, proto_compiler, exec_compatible_with = []): native.toolchain( name = name + "_toolchain", - toolchain_type = "@rules_proto//proto:toolchain_type", + toolchain_type = toolchains.PROTO_TOOLCHAIN, exec_compatible_with = exec_compatible_with, target_compatible_with = [], toolchain = name, diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 97f95e3ac4..0c90a525d4 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -120,17 +120,6 @@ def protobuf_deps(): sha256 = "160d1ebf33763124766fb35316329d907ca67f733238aa47624a8e3ff3cf2ef4", ) - # TODO: remove after toolchain types are moved to protobuf - if not native.existing_rule("rules_proto"): - http_archive( - name = "rules_proto", - sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", - strip_prefix = "rules_proto-5.3.0-21.7", - urls = [ - "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz", - ], - ) - if not native.existing_rule("proto_bazel_features"): proto_bazel_features(name = "proto_bazel_features") From ecb8214f22500ac4f514a77cb158ad13226a9b2d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 15:14:33 -0700 Subject: [PATCH 056/107] Update failure lists to include a small comment of 'why' a test failed. Makes it easier to recognize what the intended behavior should be when wanting to fix these test failures. PiperOrigin-RevId: 654912513 --- conformance/binary_json_conformance_suite.cc | 103 +++--- conformance/conformance.proto | 11 +- conformance/conformance_python.py | 8 +- conformance/conformance_test.cc | 164 +++++++--- conformance/conformance_test.h | 55 ++-- conformance/conformance_test_runner.cc | 40 ++- conformance/failure_list_cpp.txt | 204 ++++++------ conformance/failure_list_java.txt | 292 +++++++++--------- conformance/failure_list_java_lite.txt | 16 +- conformance/failure_list_objc.txt | 8 +- conformance/text_format_conformance_suite.cc | 39 ++- conformance/text_format_failure_list_cpp.txt | 82 ++--- conformance/text_format_failure_list_java.txt | 40 +-- .../text_format_failure_list_python.txt | 194 ++++++------ .../text_format_failure_list_python_cpp.txt | 144 ++++----- .../text_format_failure_list_python_upb.txt | 144 ++++----- conformance/update_failure_list.py | 44 ++- 17 files changed, 891 insertions(+), 697 deletions(-) diff --git a/conformance/binary_json_conformance_suite.cc b/conformance/binary_json_conformance_suite.cc index f631b9adad..de575bbfdd 100644 --- a/conformance/binary_json_conformance_suite.cc +++ b/conformance/binary_json_conformance_suite.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "json/reader.h" +#include "json/value.h" #include "conformance/conformance.pb.h" #include "conformance_test.h" #include "conformance/test_protos/test_messages_edition2023.pb.h" @@ -43,6 +46,7 @@ using conformance::ConformanceRequest; using conformance::ConformanceResponse; +using conformance::TestStatus; using conformance::WireFormat; using google::protobuf::Descriptor; using google::protobuf::FieldDescriptor; @@ -302,19 +306,22 @@ bool BinaryAndJsonConformanceSuite::ParseResponse( const std::string& test_name = setting.GetTestName(); ConformanceLevel level = setting.GetLevel(); + TestStatus test; + test.set_name(test_name); switch (response.result_case()) { case ConformanceResponse::kProtobufPayload: { if (requested_output != conformance::PROTOBUF) { - ReportFailure(test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided PROTOBUF instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided PROTOBUF instead.")); + ReportFailure(test, level, request, response); return false; } if (!test_message->ParseFromString(response.protobuf_payload())) { - ReportFailure(test_name, level, request, response, - "Protobuf output we received from test was unparseable."); + test.set_failure_message( + "Protobuf output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -323,16 +330,17 @@ bool BinaryAndJsonConformanceSuite::ParseResponse( case ConformanceResponse::kJsonPayload: { if (requested_output != conformance::JSON) { - ReportFailure(test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided JSON instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided JSON instead.")); + ReportFailure(test, level, request, response); return false; } if (!ParseJsonResponse(response, test_message)) { - ReportFailure(test_name, level, request, response, - "JSON output we received from test was unparseable."); + test.set_failure_message( + "JSON output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -438,13 +446,15 @@ void BinaryAndJsonConformanceSuiteImpl:: setting.GetSyntaxIdentifier(), ".ProtobufInput.", test_name); suite_.RunTest(effective_test_name, request, &response); + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to parse, but didn't."); + test.set_failure_message("Should have failed to parse, but didn't."); + suite_.ReportFailure(test, level, request, response); } } @@ -633,32 +643,34 @@ void BinaryAndJsonConformanceSuiteImpl< suite_.RunTest(effective_test_name, request, &response); + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); return; } if (response.result_case() != ConformanceResponse::kJsonPayload) { - suite_.ReportFailure(effective_test_name, level, request, response, - absl::StrCat("Expected JSON payload but got type ", - response.result_case())); + test.set_failure_message(absl::StrCat("Expected JSON payload but got type ", + response.result_case())); + suite_.ReportFailure(test, level, request, response); return; } Json::Reader reader; Json::Value value; if (!reader.parse(response.json_payload(), value)) { - suite_.ReportFailure( - effective_test_name, level, request, response, + test.set_failure_message( absl::StrCat("JSON payload cannot be parsed as valid JSON: ", reader.getFormattedErrorMessages())); + suite_.ReportFailure(test, level, request, response); return; } if (!validator(value)) { - suite_.ReportFailure(effective_test_name, level, request, response, - "JSON payload validation failed."); + test.set_failure_message("JSON payload validation failed."); + suite_.ReportFailure(test, level, request, response); return; } - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } template @@ -678,13 +690,16 @@ void BinaryAndJsonConformanceSuiteImpl::ExpectParseFailureForJson( SyntaxIdentifier(), ".JsonInput.", test_name); suite_.RunTest(effective_test_name, request, &response); + + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to parse, but didn't."); + test.set_failure_message("Should have failed to parse, but didn't."); + suite_.ReportFailure(test, level, request, response); } } @@ -708,13 +723,16 @@ void BinaryAndJsonConformanceSuiteImpl:: SyntaxIdentifier(), ".", test_name, ".JsonOutput"); suite_.RunTest(effective_test_name, request, &response); + + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kSerializeError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to serialize, but didn't."); + test.set_failure_message("Should have failed to serialize, but didn't."); + suite_.ReportFailure(test, level, request, response); } } @@ -1207,7 +1225,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( { // Tests oneof with default value. - const std::string proto = default_value; + const std::string& proto = default_value; MessageType test_message; test_message.MergeFromString(proto); std::string text; @@ -1223,7 +1241,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( { // Tests oneof with non-default value. - const std::string proto = non_default_value; + const std::string& proto = non_default_value; MessageType test_message; test_message.MergeFromString(proto); std::string text; @@ -1240,7 +1258,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( { // Tests oneof with multiple values of the same field. const std::string proto = absl::StrCat(default_value, non_default_value); - const std::string expected_proto = non_default_value; + const std::string& expected_proto = non_default_value; MessageType test_message; test_message.MergeFromString(expected_proto); std::string text; @@ -1266,7 +1284,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( GetDefaultValue(other_type)); const std::string proto = absl::StrCat(other_value, non_default_value); - const std::string expected_proto = non_default_value; + const std::string& expected_proto = non_default_value; MessageType test_message; test_message.MergeFromString(expected_proto); std::string text; @@ -1421,9 +1439,12 @@ void BinaryAndJsonConformanceSuiteImpl::TestUnknownOrdering() { const ConformanceRequest& request = setting.GetRequest(); ConformanceResponse response; suite_.RunTest(setting.GetTestName(), request, &response); + MessageType response_message; + TestStatus test; + test.set_name(setting.GetTestName()); if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(setting.GetTestName(), request, response); + suite_.ReportSkip(test, request, response); return; } suite_.ParseResponse(response, setting, &response_message); @@ -1441,10 +1462,10 @@ void BinaryAndJsonConformanceSuiteImpl::TestUnknownOrdering() { ufs.field(1).varint() != 123 || ufs.field(2).length_delimited() != "def" || ufs.field(3).varint() != 456) { - suite_.ReportFailure(setting.GetTestName(), setting.GetLevel(), request, - response, "Unknown field mismatch"); + test.set_failure_message("Unknown field mismatch"); + suite_.ReportFailure(test, setting.GetLevel(), request, response); } else { - suite_.ReportSuccess(setting.GetTestName()); + suite_.ReportSuccess(test); } } diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 2357d59ad6..a53b5e21d4 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -57,11 +57,20 @@ enum TestCategory { TEXT_FORMAT_TEST = 5; } +// Meant to encapsulate all types of tests: successes, skips, failures, etc. +// Therefore, this may or may not have a failure message. Failure messages +// may be truncated for our failure lists. +message TestStatus { + string name = 1; + string failure_message = 2; +} + // The conformance runner will request a list of failures as the first request. // This will be known by message_type == "conformance.FailureSet", a conformance // test should return a serialized FailureSet in protobuf_payload. message FailureSet { - repeated string failure = 1; + repeated TestStatus test = 2; + reserved 1; } // Represents a single test case's input. The testee should: diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py index 60182e1e9f..eea05d9fe7 100755 --- a/conformance/conformance_python.py +++ b/conformance/conformance_python.py @@ -98,7 +98,7 @@ def do_test(request): "Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64", ] for x in failures: - failure_set.failure.append(x) + failure_set.test.append(conformance_pb2.TestStatus(name=x)) response.protobuf_payload = failure_set.SerializeToString() return response @@ -194,6 +194,8 @@ def do_test_io(): while True: if not do_test_io(): - sys.stderr.write("conformance_python: received EOF from test runner " + - "after %s tests, exiting\n" % (test_count)) + sys.stderr.write( + "conformance_python: received EOF from test runner " + + "after %s tests, exiting\n" % (test_count,) + ) sys.exit(0) diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 0f584b95a3..10b3103ae1 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -9,14 +9,18 @@ #include +#include #include #include +#include #include #include #include +#include #include "google/protobuf/util/field_comparator.h" #include "google/protobuf/util/message_differencer.h" +#include "absl/container/btree_map.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" @@ -30,6 +34,7 @@ using conformance::ConformanceRequest; using conformance::ConformanceResponse; +using conformance::TestStatus; using conformance::WireFormat; using google::protobuf::util::DefaultFieldComparator; using google::protobuf::util::MessageDifferencer; @@ -52,16 +57,37 @@ static std::string ToOctString(const std::string& binary_string) { return oct_string; } -template -bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file, - absl::string_view msg, absl::string_view output_dir, - std::string* output) { +// Removes all newlines. +static void Normalize(std::string& input) { + input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); +} + +// Sets up a failure message properly for our failure lists. +static TestStatus FormatFailureMessage(TestStatus& input) { + // Make copy just this once, as we need to modify it for our failure lists. + std::string formatted_failure_message = input.failure_message(); + // Remove newlines + Normalize(formatted_failure_message); + // Truncate failure message if needed + if (formatted_failure_message.length() > 128) { + formatted_failure_message = formatted_failure_message.substr(0, 128); + } + TestStatus properly_formatted; + properly_formatted.set_name(input.name()); + properly_formatted.set_failure_message(formatted_failure_message); + return properly_formatted; +} + +bool CheckSetEmpty(const absl::btree_map& set_to_check, + absl::string_view write_to_file, absl::string_view msg, + absl::string_view output_dir, std::string* output) { if (set_to_check.empty()) return true; absl::StrAppendFormat(output, "\n"); absl::StrAppendFormat(output, "%s\n\n", msg); - for (absl::string_view v : set_to_check) { - absl::StrAppendFormat(output, " %s\n", v); + for (const auto& pair : set_to_check) { + absl::StrAppendFormat(output, " %s # %s\n", pair.first, + pair.second.failure_message()); } absl::StrAppendFormat(output, "\n"); @@ -75,8 +101,8 @@ bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file, } std::ofstream os{std::string(filename)}; if (os) { - for (absl::string_view v : set_to_check) { - os << v << "\n"; + for (const auto& pair : set_to_check) { + os << pair.first << " # " << pair.second.failure_message() << "\n"; } } else { absl::StrAppendFormat(output, @@ -264,47 +290,72 @@ ConformanceResponse ConformanceTestSuite::TruncateResponse( return debug_response; } -void ConformanceTestSuite::ReportSuccess(const std::string& test_name) { - if (expected_to_fail_.erase(test_name) != 0) { +void ConformanceTestSuite::ReportSuccess(const TestStatus& test) { + if (expected_to_fail_.erase(test.name()) != 0) { absl::StrAppendFormat( &output_, "ERROR: test %s is in the failure list, but test succeeded. " "Remove it from the failure list.\n", - test_name); - unexpected_succeeding_tests_.insert(test_name); + test.name()); + unexpected_succeeding_tests_[test.name()] = test; } successes_++; } -void ConformanceTestSuite::ReportFailure(const std::string& test_name, +void ConformanceTestSuite::ReportFailure(TestStatus& test, ConformanceLevel level, const ConformanceRequest& request, - const ConformanceResponse& response, - absl::string_view message) { - if (expected_to_fail_.erase(test_name) == 1) { - expected_failures_++; + const ConformanceResponse& response) { + if (expected_to_fail_.contains(test.name())) { + // Make copy just this once, as we need to modify them for comparison. + // Failure message from the failure list. + string expected_failure_message = + expected_to_fail_[test.name()].failure_message(); + // Actual failure message from the test run. + std::string actual_failure_message = test.failure_message(); + + Normalize(actual_failure_message); + if (actual_failure_message.rfind(expected_failure_message, 0) == 0) { + // Our failure messages match. + expected_failures_++; + } else { + // We want to add the test to the failure list with its correct failure + // message. + unexpected_failure_messages_[test.name()] = FormatFailureMessage(test); + // We want to remove the test from the failure list. That means passing + // to it the same failure message that was in the list. + TestStatus incorrect_failure_message; + incorrect_failure_message.set_name(test.name()); + incorrect_failure_message.set_failure_message( + expected_to_fail_[test.name()].failure_message()); + + expected_failure_messages_[test.name()] = incorrect_failure_message; + } + expected_to_fail_.erase(test.name()); if (!verbose_) return; } else if (level == RECOMMENDED && !enforce_recommended_) { - absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test_name); + absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test.name()); } else { - absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test_name); - unexpected_failing_tests_.insert(test_name); + absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test.name()); + + unexpected_failing_tests_[test.name()] = FormatFailureMessage(test); } - absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", message, + absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", + test.failure_message(), TruncateRequest(request).ShortDebugString(), TruncateResponse(response).ShortDebugString()); } -void ConformanceTestSuite::ReportSkip(const std::string& test_name, +void ConformanceTestSuite::ReportSkip(const TestStatus& test, const ConformanceRequest& request, const ConformanceResponse& response) { if (verbose_) { absl::StrAppendFormat( - &output_, "SKIPPED, test=%s request=%s, response=%s\n", test_name, + &output_, "SKIPPED, test=%s request=%s, response=%s\n", test.name(), request.ShortDebugString(), response.ShortDebugString()); } - skipped_.insert(test_name); + skipped_[test.name()] = test; } void ConformanceTestSuite::RunValidInputTest( @@ -344,22 +395,26 @@ void ConformanceTestSuite::VerifyResponse( ABSL_CHECK(reference_message->ParseFromString(equivalent_wire_format)) << "Failed to parse wire data for test case: " << test_name; + TestStatus test; + test.set_name(test_name); + switch (response.result_case()) { case ConformanceResponse::RESULT_NOT_SET: - ReportFailure(test_name, level, request, response, - "Response didn't have any field in the Response."); + test.set_failure_message( + "Response didn't have any field in the Response."); + ReportFailure(test, level, request, response); return; case ConformanceResponse::kParseError: case ConformanceResponse::kTimeoutError: case ConformanceResponse::kRuntimeError: case ConformanceResponse::kSerializeError: - ReportFailure(test_name, level, request, response, - "Failed to parse input or produce output."); + test.set_failure_message("Failed to parse input or produce output."); + ReportFailure(test, level, request, response); return; case ConformanceResponse::kSkipped: - ReportSkip(test_name, request, response); + ReportSkip(test, request, response); return; default: @@ -385,16 +440,14 @@ void ConformanceTestSuite::VerifyResponse( } else { check = differencer.Compare(*reference_message, *test_message); } - if (check) { if (need_report_success) { - ReportSuccess(test_name); + ReportSuccess(test); } } else { - ReportFailure( - test_name, level, request, response, - absl::StrCat("Output was not equivalent to reference message: ", - differences)); + test.set_failure_message(absl::StrCat( + "Output was not equivalent to reference message: ", differences)); + ReportFailure(test, level, request, response); } } @@ -442,8 +495,8 @@ std::string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) { return ""; } -void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) { - expected_to_fail_.insert(test_name); +void ConformanceTestSuite::AddExpectedFailedTest(const TestStatus& failure) { + expected_to_fail_[failure.name()] = failure; } bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, @@ -462,7 +515,7 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, failure_list_filename_ = filename; expected_to_fail_.clear(); - for (const std::string& failure : failure_list->failure()) { + for (const TestStatus& failure : failure_list->test()) { AddExpectedFailedTest(failure); } RunSuiteImpl(); @@ -485,6 +538,35 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, output_dir_, &output_)) { ok = false; } + if (!CheckSetEmpty( + expected_failure_messages_, "expected_failure_messages.txt", + absl::StrCat( + "These tests were listed in the failure list, but their failure " + "messages do not match. Remove them from the failure list " + "by running:\n" + " bazel run ", + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --remove ", output_dir_, + "expected_failure_messages.txt"), + output_dir_, &output_)) { + ok = false; + } + if (!CheckSetEmpty( + unexpected_failure_messages_, "unexpected_failure_messages.txt", + absl::StrCat( + "These tests failed because their failure messages did " + "not match. If they can't be fixed right now, " + "you can add them to the failure list so the overall " + "suite can succeed. Add them to the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --add ", output_dir_, + "unexpected_failure_messages.txt"), + output_dir_, &output_)) { + ok = false; + } + if (!CheckSetEmpty( unexpected_failing_tests_, "failing_tests.txt", absl::StrCat( @@ -522,9 +604,11 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, absl::StrAppendFormat(&output_, "CONFORMANCE SUITE %s: %d successes, %zu skipped, " - "%d expected failures, %zu unexpected failures.\n", + "%d expected failures, %zu unexpected failures, %zu " + "unexpected_failure_messages.\n", ok ? "PASSED" : "FAILED", successes_, skipped_.size(), - expected_failures_, unexpected_failing_tests_.size()); + expected_failures_, unexpected_failing_tests_.size(), + unexpected_failure_messages_.size()); absl::StrAppendFormat(&output_, "\n"); output->assign(output_); diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index c78f9ea8ab..2785138482 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -20,13 +20,11 @@ #include #include "google/protobuf/descriptor.pb.h" -#include "google/protobuf/util/type_resolver.h" -#include "absl/container/btree_set.h" +#include "absl/container/btree_map.h" #include "absl/container/flat_hash_set.h" -#include "absl/strings/string_view.h" +#include "conformance/conformance.pb.h" #include "conformance/conformance.pb.h" #include "google/protobuf/descriptor.h" -#include "google/protobuf/wire_format_lite.h" namespace conformance { class ConformanceRequest; @@ -46,7 +44,7 @@ class ConformanceTestSuite; class ConformanceTestRunner { public: - virtual ~ConformanceTestRunner() {} + virtual ~ConformanceTestRunner() = default; // Call to run a single conformance test. // @@ -78,10 +76,10 @@ class ForkPipeRunner : public ConformanceTestRunner { explicit ForkPipeRunner(const std::string& executable) : child_pid_(-1), executable_(executable) {} - virtual ~ForkPipeRunner() {} + ~ForkPipeRunner() override = default; void RunTest(const std::string& test_name, const std::string& request, - std::string* response); + std::string* response) override; private: void SpawnTestProgram(); @@ -134,7 +132,7 @@ class ConformanceTestSuite { enforce_recommended_(false), maximum_edition_(Edition::EDITION_PROTO3), failure_list_flag_name_("--failure_list") {} - virtual ~ConformanceTestSuite() {} + virtual ~ConformanceTestSuite() = default; void SetPerformance(bool performance) { performance_ = performance; } void SetVerbose(bool verbose) { verbose_ = verbose; } @@ -201,7 +199,7 @@ class ConformanceTestSuite { const Message& prototype_message, const std::string& test_name, const std::string& input); - virtual ~ConformanceRequestSetting() {} + virtual ~ConformanceRequestSetting() = default; std::unique_ptr NewTestMessage() const; @@ -259,12 +257,11 @@ class ConformanceTestSuite { conformance::ConformanceResponse TruncateResponse( const conformance::ConformanceResponse& response); - void ReportSuccess(const std::string& test_name); - void ReportFailure(const std::string& test_name, ConformanceLevel level, + void ReportSuccess(const conformance::TestStatus& test); + void ReportFailure(conformance::TestStatus& test, ConformanceLevel level, const conformance::ConformanceRequest& request, - const conformance::ConformanceResponse& response, - absl::string_view message); - void ReportSkip(const std::string& test_name, + const conformance::ConformanceResponse& response); + void ReportSkip(const conformance::TestStatus& test, const conformance::ConformanceRequest& request, const conformance::ConformanceResponse& response); @@ -278,7 +275,7 @@ class ConformanceTestSuite { const conformance::ConformanceRequest& request, conformance::ConformanceResponse* response); - void AddExpectedFailedTest(const std::string& test_name); + void AddExpectedFailedTest(const conformance::TestStatus& failure); virtual void RunSuiteImpl() = 0; @@ -296,20 +293,36 @@ class ConformanceTestSuite { // The set of test names that are expected to fail in this run, but haven't // failed yet. - absl::btree_set expected_to_fail_; + absl::btree_map expected_to_fail_; + + // The set of tests that failed because their failure message did not match + // the actual failure message. These are failure messages that may need to be + // removed from our failure lists. + absl::btree_map + expected_failure_messages_; // The set of test names that have been run. Used to ensure that there are no // duplicate names in the suite. absl::flat_hash_set test_names_; - // The set of tests that failed, but weren't expected to. - absl::btree_set unexpected_failing_tests_; + // The set of tests that failed, but weren't expected to: They weren't + // present in our failure lists. + absl::btree_map + unexpected_failing_tests_; + + // The set of tests that succeeded, but weren't expected to: They were present + // in our failure lists, but managed to succeed. + absl::btree_map + unexpected_succeeding_tests_; - // The set of tests that succeeded, but weren't expected to. - absl::btree_set unexpected_succeeding_tests_; + // The set of tests that failed because their failure message did not match + // the actual failure message. These are failure messages that may need to be + // added to our failure lists. + absl::btree_map + unexpected_failure_messages_; // The set of tests that the testee opted out of; - absl::btree_set skipped_; + absl::btree_map skipped_; }; } // namespace protobuf diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index 35c33c63d7..49aaea374b 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -37,21 +37,24 @@ #include #include +#include +#include #include #include #include #include #include +#include #include #include "absl/log/absl_log.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" +#include "conformance/conformance.pb.h" #include "conformance/conformance.pb.h" #include "conformance_test.h" #include "google/protobuf/endian.h" -using conformance::ConformanceResponse; using google::protobuf::ConformanceTestSuite; using std::string; using std::vector; @@ -76,15 +79,34 @@ void ParseFailureList(const char *filename, exit(1); } - for (string line; getline(infile, line);) { - // Remove whitespace. - line.erase(std::remove_if(line.begin(), line.end(), ::isspace), line.end()); - + for (string line; std::getline(infile, line);) { // Remove comments. - line = line.substr(0, line.find("#")); + string test_name = line.substr(0, line.find('#')); + + test_name.erase( + std::remove_if(test_name.begin(), test_name.end(), ::isspace), + test_name.end()); - if (!line.empty()) { - failure_list->add_failure(line); + if (test_name.empty()) { // Skip empty lines. + continue; + } + + // If we remove whitespace from the beginning of a line, and what we have + // left at first is a '#', then we have a comment. + if (test_name[0] != '#') { + // Find our failure message if it exists. Will be set to an empty string + // if no message is found. Empty failure messages also pass our tests. + size_t check_message = line.find('#'); + string message; + if (check_message != std::string::npos) { + message = line.substr(check_message + 1); // +1 to skip the delimiter + // If we had only whitespace after the delimiter, we will have an empty + // failure message and the test will still pass. + message = std::string(absl::StripAsciiWhitespace(message)); + } + conformance::TestStatus *test = failure_list->add_test(); + test->set_name(test_name); + test->set_failure_message(message); } } } diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index 0f4e10ff63..d7ffba7274 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt @@ -7,105 +7,105 @@ # TODO: insert links to corresponding bugs tracking the issue. # Should we use GitHub issues or the Google-internal bug tracker? -Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Proto3.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Proto3.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted -Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Proto3.JsonInput.MapFieldValueIsNull -Recommended.Editions_Proto3.JsonInput.MapFieldValueIsNull -Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Editions_Proto3.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Editions_Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Proto3.JsonInput.RepeatedFieldTrailingComma -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingComma -Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto3.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Editions_Proto3.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Proto3.JsonInput.TrailingCommaInAnObject -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObject -Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace -Recommended.Proto2.JsonInput.FieldNameExtension.Validator -Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Editions_Proto2.JsonInput.MapFieldValueIsNull -Recommended.Editions_Proto2.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Editions_Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingComma -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto2.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObject -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto2.JsonInput.FieldNameDuplicate -Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Proto2.JsonInput.FieldNameNotQuoted -Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Proto2.JsonInput.MapFieldValueIsNull -Recommended.Proto2.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Proto2.JsonInput.RepeatedFieldTrailingComma -Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto2.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Proto2.JsonInput.TrailingCommaInAnObject -Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Editions_Proto2.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Editions_Proto3.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Proto2.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Proto3.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt index ef705ef1b8..d8f44fde1f 100644 --- a/conformance/failure_list_java.txt +++ b/conformance/failure_list_java.txt @@ -4,149 +4,149 @@ # By listing them here we can keep tabs on which ones are failing and be sure # that we don't introduce regressions in other tests. -Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Proto3.JsonInput.BoolFieldAllCapitalFalse -Recommended.Proto3.JsonInput.BoolFieldAllCapitalTrue -Recommended.Proto3.JsonInput.BoolFieldCamelCaseFalse -Recommended.Proto3.JsonInput.BoolFieldCamelCaseTrue -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto3.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted -Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Proto3.JsonInput.FieldNameDuplicate -Recommended.Proto3.JsonInput.FieldNameNotQuoted -Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted -Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Proto3.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.JsonWithComments -Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto3.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.FieldNameExtension.Validator -Required.Proto3.JsonInput.EnumFieldNotQuoted -Required.Proto3.JsonInput.Int32FieldLeadingZero -Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Proto3.JsonInput.Int32FieldPlusSign -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Proto3.JsonInput.StringFieldNotAString -Recommended.Proto2.JsonInput.BoolFieldAllCapitalFalse -Recommended.Proto2.JsonInput.BoolFieldAllCapitalTrue -Recommended.Proto2.JsonInput.BoolFieldCamelCaseFalse -Recommended.Proto2.JsonInput.BoolFieldCamelCaseTrue -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto2.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Proto2.JsonInput.DoubleFieldNanNotQuoted -Recommended.Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Proto2.JsonInput.FieldNameDuplicate -Recommended.Proto2.JsonInput.FieldNameNotQuoted -Recommended.Proto2.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Proto2.JsonInput.FloatFieldNanNotQuoted -Recommended.Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Proto2.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.JsonWithComments -Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto2.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Proto2.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Proto2.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Proto2.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.Uint64MapFieldKeyNotQuoted -Required.Proto2.JsonInput.EnumFieldNotQuoted -Required.Proto2.JsonInput.Int32FieldLeadingZero -Required.Proto2.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Proto2.JsonInput.Int32FieldPlusSign -Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Proto2.JsonInput.StringFieldNotAString -Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalTrue -Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseTrue -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto3.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.DoubleFieldNanNotQuoted -Recommended.Editions_Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto3.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.FloatFieldNanNotQuoted -Recommended.Editions_Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.JsonWithComments -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto3.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Editions_Proto3.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.Uint64MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator -Required.Editions_Proto3.JsonInput.EnumFieldNotQuoted -Required.Editions_Proto3.JsonInput.Int32FieldLeadingZero -Required.Editions_Proto3.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Editions_Proto3.JsonInput.Int32FieldPlusSign -Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Editions_Proto3.JsonInput.StringFieldNotAString -Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalTrue -Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseTrue -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto2.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.DoubleFieldNanNotQuoted -Recommended.Editions_Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto2.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.FloatFieldNanNotQuoted -Recommended.Editions_Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.JsonWithComments -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto2.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Editions_Proto2.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.Uint64MapFieldKeyNotQuoted -Required.Editions_Proto2.JsonInput.EnumFieldNotQuoted -Required.Editions_Proto2.JsonInput.Int32FieldLeadingZero -Required.Editions_Proto2.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Editions_Proto2.JsonInput.Int32FieldPlusSign -Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Editions_Proto2.JsonInput.StringFieldNotAString -Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput +Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Editions_Proto3.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto2.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto3.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch \ No newline at end of file diff --git a/conformance/failure_list_java_lite.txt b/conformance/failure_list_java_lite.txt index 84606a5173..87a9758cba 100644 --- a/conformance/failure_list_java_lite.txt +++ b/conformance/failure_list_java_lite.txt @@ -4,11 +4,11 @@ # By listing them here we can keep tabs on which ones are failing and be sure # that we don't introduce regressions in other tests. -Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE \ No newline at end of file +Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. \ No newline at end of file diff --git a/conformance/failure_list_objc.txt b/conformance/failure_list_objc.txt index 463c3606b9..0dd4279587 100644 --- a/conformance/failure_list_objc.txt +++ b/conformance/failure_list_objc.txt @@ -1,7 +1,7 @@ # JSON input or output tests are skipped (in conformance_objc.m) as mobile # platforms don't support JSON wire format to avoid code bloat. -Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput +Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index a230fa3d08..2f6a609fa0 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -26,6 +26,7 @@ using conformance::ConformanceRequest; using conformance::ConformanceResponse; +using conformance::TestStatus; using conformance::WireFormat; using protobuf_test_messages::editions::TestAllTypesEdition2023; using protobuf_test_messages::proto2::TestAllTypesProto2; @@ -73,19 +74,22 @@ bool TextFormatConformanceTestSuite::ParseResponse( const std::string& test_name = setting.GetTestName(); ConformanceLevel level = setting.GetLevel(); + TestStatus test; + test.set_name(test_name); switch (response.result_case()) { case ConformanceResponse::kProtobufPayload: { if (requested_output != conformance::PROTOBUF) { - ReportFailure(test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided PROTOBUF instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided PROTOBUF instead.")); + ReportFailure(test, level, request, response); return false; } if (!test_message->ParseFromString(response.protobuf_payload())) { - ReportFailure(test_name, level, request, response, - "Protobuf output we received from test was unparseable."); + test.set_failure_message( + "Protobuf output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -94,18 +98,17 @@ bool TextFormatConformanceTestSuite::ParseResponse( case ConformanceResponse::kTextPayload: { if (requested_output != conformance::TEXT_FORMAT) { - ReportFailure( - test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided TEXT_FORMAT instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided TEXT_FORMAT instead.")); + ReportFailure(test, level, request, response); return false; } if (!ParseTextFormatResponse(response, setting, test_message)) { - ReportFailure( - test_name, level, request, response, + test.set_failure_message( "TEXT_FORMAT output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -173,13 +176,15 @@ void TextFormatConformanceTestSuiteImpl::ExpectParseFailure( setting.GetSyntaxIdentifier(), ".TextFormatInput.", test_name); suite_.RunTest(effective_test_name, request, &response); + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to parse, but didn't."); + test.set_failure_message("Should have failed to parse, but didn't."); + suite_.ReportFailure(test, level, request, response); } } diff --git a/conformance/text_format_failure_list_cpp.txt b/conformance/text_format_failure_list_cpp.txt index fd2d7ada1f..d1d3cd2e76 100644 --- a/conformance/text_format_failure_list_cpp.txt +++ b/conformance/text_format_failure_list_cpp.txt @@ -1,40 +1,42 @@ -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString -Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes -Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes -Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString -Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString # Should have failed to parse, but didn't. + +# End up setting the high bit as a sign instead of failing to parse. diff --git a/conformance/text_format_failure_list_java.txt b/conformance/text_format_failure_list_java.txt index a035453944..d2611ccb10 100644 --- a/conformance/text_format_failure_list_java.txt +++ b/conformance/text_format_failure_list_java.txt @@ -1,20 +1,20 @@ -Recommended.Editions_Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Editions_Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput -Recommended.Editions_Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Editions_Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.AnyField.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.AnyField.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Proto3.TextFormatInput.AnyField.ProtobufOutput -Required.Proto3.TextFormatInput.AnyField.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal +Recommended.Editions_Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Editions_Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Editions_Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Editions_Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Required.Editions_Proto3.TextFormatInput.AnyField.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.AnyField.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.AnyField.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.AnyField.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. diff --git a/conformance/text_format_failure_list_python.txt b/conformance/text_format_failure_list_python.txt index 48abf4e55b..346852001c 100644 --- a/conformance/text_format_failure_list_python.txt +++ b/conformance/text_format_failure_list_python.txt @@ -1,99 +1,101 @@ # This is the list of text format conformance tests that are known to fail right # now. # TODO: These should be fixed. -Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput -Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput -Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ + +# Optional float interpreted as `inf` +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput # Output was not equivalent to reference message +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput # Output was not equivalent to reference message +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput # Output was not equivalent to reference message +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput # Output was not equivalent to reference message diff --git a/conformance/text_format_failure_list_python_cpp.txt b/conformance/text_format_failure_list_python_cpp.txt index fc233ed286..d3c827d8d5 100644 --- a/conformance/text_format_failure_list_python_cpp.txt +++ b/conformance/text_format_failure_list_python_cpp.txt @@ -1,72 +1,72 @@ -Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ diff --git a/conformance/text_format_failure_list_python_upb.txt b/conformance/text_format_failure_list_python_upb.txt index 85da8a1660..476a557e27 100644 --- a/conformance/text_format_failure_list_python_upb.txt +++ b/conformance/text_format_failure_list_python_upb.txt @@ -1,75 +1,75 @@ # This is the list of text format conformance tests that are known to fail right # now. # TODO: These should be fixed. -Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ diff --git a/conformance/update_failure_list.py b/conformance/update_failure_list.py index 639a8c26ed..917dba312b 100755 --- a/conformance/update_failure_list.py +++ b/conformance/update_failure_list.py @@ -8,6 +8,14 @@ """Script to update a failure list file to add/remove failures. +When adding, will attempt to place them in their correct lexicographical +position relative to other test names. This requires that the failure list is +already sorted. This does not guarantee that the tests will appear neatly one +after the other, as there may be comments in between. If the failure list +is not sorted, sorted blocks may be produced, but the list as a whole will not. +Lastly, be wary of lists that have tests stripped from OSS; if you catch that +a test was added inside a stripped block, you may need to move it out. + This is sort of like comm(1), except it recognizes comments and ignores them. """ @@ -19,32 +27,58 @@ parser.add_argument('filename', type=str, help='failure list file to update') parser.add_argument('--add', dest='add_list', action='append') parser.add_argument('--remove', dest='remove_list', action='append') +DEFAULT_ALIGNMENT = 114 args = parser.parse_args() add_set = set() remove_set = set() +# Adds test + failure message for add_file in (args.add_list or []): with open(add_file) as f: for line in f: add_set.add(line) +# We only remove by name for remove_file in (args.remove_list or []): with open(remove_file) as f: for line in f: if line in add_set: raise Exception("Asked to both add and remove test: " + line) - remove_set.add(line.strip()) + remove_set.add(line.split('#')[0].strip()) add_list = sorted(add_set, reverse=True) with open(args.filename) as in_file: - existing_list = in_file.read() + existing_list = in_file.read() -with open(args.filename, "w") as f: +with open(args.filename, 'w') as f: for line in existing_list.splitlines(True): - test = line.split("#")[0].strip() - while len(add_list) > 0 and test > add_list[-1]: + test = line.split('#')[0].strip() + # As long as the tests we are adding appear lexicographically before our + # read test, put them first followed by our read test. + while add_list and test > add_list[-1]: f.write(add_list.pop()) if test not in remove_set: f.write(line) + # Any remaining tests are added at the end + while add_list: + f.write(add_list.pop()) + +# Update our read of the existing file +with open(args.filename, 'r') as f: + existing_list = f.read() + +# Actual alignment of failure messages to 'DEFAULT_ALIGNMENT' +# If test name exceeds DEFAULT_ALIGNMENT, it cannot and will not be aligned. +with open(args.filename, 'w') as f: + for line in existing_list.splitlines(True): + split = line.split('#', 1) + test = split[0].strip() + if len(split) > 1 and test: + message = split[1].lstrip() + spaces = ' ' * (DEFAULT_ALIGNMENT - len(test)) + line = test + spaces + ' # ' + message + f.write(line) + else: # ignore blank lines/lines that do not have '#'/comments + f.write(line) From d6dd8f2ee4c272f8114a6dac621e509ff872eb1f Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Jul 2024 22:28:55 +0000 Subject: [PATCH 057/107] Auto-generate files after cl/654912513 --- .../Conformance.pb.cs | 333 +++++++++++++++--- 1 file changed, 288 insertions(+), 45 deletions(-) diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs index f8a073b938..29b19592a7 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs @@ -25,32 +25,35 @@ namespace Conformance { byte[] descriptorData = global::System.Convert.FromBase64String( string.Concat( "Ch1jb25mb3JtYW5jZS9jb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2Ui", - "HQoKRmFpbHVyZVNldBIPCgdmYWlsdXJlGAEgAygJIuMCChJDb25mb3JtYW5j", - "ZVJlcXVlc3QSGgoQcHJvdG9idWZfcGF5bG9hZBgBIAEoDEgAEhYKDGpzb25f", - "cGF5bG9hZBgCIAEoCUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRl", - "eHRfcGF5bG9hZBgIIAEoCUgAEjgKF3JlcXVlc3RlZF9vdXRwdXRfZm9ybWF0", - "GAMgASgOMhcuY29uZm9ybWFuY2UuV2lyZUZvcm1hdBIUCgxtZXNzYWdlX3R5", - "cGUYBCABKAkSMAoNdGVzdF9jYXRlZ29yeRgFIAEoDjIZLmNvbmZvcm1hbmNl", - "LlRlc3RDYXRlZ29yeRI+ChVqc3BiX2VuY29kaW5nX29wdGlvbnMYBiABKAsy", - "Hy5jb25mb3JtYW5jZS5Kc3BiRW5jb2RpbmdDb25maWcSHAoUcHJpbnRfdW5r", - "bm93bl9maWVsZHMYCSABKAhCCQoHcGF5bG9hZCL6AQoTQ29uZm9ybWFuY2VS", - "ZXNwb25zZRIVCgtwYXJzZV9lcnJvchgBIAEoCUgAEhkKD3NlcmlhbGl6ZV9l", - "cnJvchgGIAEoCUgAEhcKDXRpbWVvdXRfZXJyb3IYCSABKAlIABIXCg1ydW50", - "aW1lX2Vycm9yGAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgA", - "EhYKDGpzb25fcGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIABIW", - "Cgxqc3BiX3BheWxvYWQYByABKAlIABIWCgx0ZXh0X3BheWxvYWQYCCABKAlI", - "AEIICgZyZXN1bHQiNwoSSnNwYkVuY29kaW5nQ29uZmlnEiEKGXVzZV9qc3Bi", - "X2FycmF5X2FueV9mb3JtYXQYASABKAgqUAoKV2lyZUZvcm1hdBIPCgtVTlNQ", - "RUNJRklFRBAAEgwKCFBST1RPQlVGEAESCAoESlNPThACEggKBEpTUEIQAxIP", - "CgtURVhUX0ZPUk1BVBAEKo8BCgxUZXN0Q2F0ZWdvcnkSFAoQVU5TUEVDSUZJ", - "RURfVEVTVBAAEg8KC0JJTkFSWV9URVNUEAESDQoJSlNPTl9URVNUEAISJAog", - "SlNPTl9JR05PUkVfVU5LTk9XTl9QQVJTSU5HX1RFU1QQAxINCglKU1BCX1RF", - "U1QQBBIUChBURVhUX0ZPUk1BVF9URVNUEAVCLwofY29tLmdvb2dsZS5wcm90", - "b2J1Zi5jb25mb3JtYW5jZaICC0NvbmZvcm1hbmNlYgZwcm90bzM=")); + "MwoKVGVzdFN0YXR1cxIMCgRuYW1lGAEgASgJEhcKD2ZhaWx1cmVfbWVzc2Fn", + "ZRgCIAEoCSI5CgpGYWlsdXJlU2V0EiUKBHRlc3QYAiADKAsyFy5jb25mb3Jt", + "YW5jZS5UZXN0U3RhdHVzSgQIARACIuMCChJDb25mb3JtYW5jZVJlcXVlc3QS", + "GgoQcHJvdG9idWZfcGF5bG9hZBgBIAEoDEgAEhYKDGpzb25fcGF5bG9hZBgC", + "IAEoCUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRleHRfcGF5bG9h", + "ZBgIIAEoCUgAEjgKF3JlcXVlc3RlZF9vdXRwdXRfZm9ybWF0GAMgASgOMhcu", + "Y29uZm9ybWFuY2UuV2lyZUZvcm1hdBIUCgxtZXNzYWdlX3R5cGUYBCABKAkS", + "MAoNdGVzdF9jYXRlZ29yeRgFIAEoDjIZLmNvbmZvcm1hbmNlLlRlc3RDYXRl", + "Z29yeRI+ChVqc3BiX2VuY29kaW5nX29wdGlvbnMYBiABKAsyHy5jb25mb3Jt", + "YW5jZS5Kc3BiRW5jb2RpbmdDb25maWcSHAoUcHJpbnRfdW5rbm93bl9maWVs", + "ZHMYCSABKAhCCQoHcGF5bG9hZCL6AQoTQ29uZm9ybWFuY2VSZXNwb25zZRIV", + "CgtwYXJzZV9lcnJvchgBIAEoCUgAEhkKD3NlcmlhbGl6ZV9lcnJvchgGIAEo", + "CUgAEhcKDXRpbWVvdXRfZXJyb3IYCSABKAlIABIXCg1ydW50aW1lX2Vycm9y", + "GAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgAEhYKDGpzb25f", + "cGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIABIWCgxqc3BiX3Bh", + "eWxvYWQYByABKAlIABIWCgx0ZXh0X3BheWxvYWQYCCABKAlIAEIICgZyZXN1", + "bHQiNwoSSnNwYkVuY29kaW5nQ29uZmlnEiEKGXVzZV9qc3BiX2FycmF5X2Fu", + "eV9mb3JtYXQYASABKAgqUAoKV2lyZUZvcm1hdBIPCgtVTlNQRUNJRklFRBAA", + "EgwKCFBST1RPQlVGEAESCAoESlNPThACEggKBEpTUEIQAxIPCgtURVhUX0ZP", + "Uk1BVBAEKo8BCgxUZXN0Q2F0ZWdvcnkSFAoQVU5TUEVDSUZJRURfVEVTVBAA", + "Eg8KC0JJTkFSWV9URVNUEAESDQoJSlNPTl9URVNUEAISJAogSlNPTl9JR05P", + "UkVfVU5LTk9XTl9QQVJTSU5HX1RFU1QQAxINCglKU1BCX1RFU1QQBBIUChBU", + "RVhUX0ZPUk1BVF9URVNUEAVCLwofY29tLmdvb2dsZS5wcm90b2J1Zi5jb25m", + "b3JtYW5jZaICC0NvbmZvcm1hbmNlYgZwcm90bzM=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), typeof(global::Conformance.TestCategory), }, null, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.FailureSet), global::Conformance.FailureSet.Parser, new[]{ "Failure" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.TestStatus), global::Conformance.TestStatus.Parser, new[]{ "Name", "FailureMessage" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.FailureSet), global::Conformance.FailureSet.Parser, new[]{ "Test" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "JspbPayload", "TextPayload", "RequestedOutputFormat", "MessageType", "TestCategory", "JspbEncodingOptions", "PrintUnknownFields" }, new[]{ "Payload" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceResponse), global::Conformance.ConformanceResponse.Parser, new[]{ "ParseError", "SerializeError", "TimeoutError", "RuntimeError", "ProtobufPayload", "JsonPayload", "Skipped", "JspbPayload", "TextPayload" }, new[]{ "Result" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.JspbEncodingConfig), global::Conformance.JspbEncodingConfig.Parser, new[]{ "UseJspbArrayAnyFormat" }, null, null, null, null) @@ -104,6 +107,246 @@ namespace Conformance { #endregion #region Messages + /// + /// Meant to encapsulate all types of tests: successes, skips, failures, etc. + /// Therefore, this may or may not have a failure message. Failure messages + /// may be truncated for our failure lists. + /// + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class TestStatus : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new TestStatus()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestStatus() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestStatus(TestStatus other) : this() { + name_ = other.name_; + failureMessage_ = other.failureMessage_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestStatus Clone() { + return new TestStatus(this); + } + + /// Field number for the "name" field. + public const int NameFieldNumber = 1; + private string name_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Name { + get { return name_; } + set { + name_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "failure_message" field. + public const int FailureMessageFieldNumber = 2; + private string failureMessage_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string FailureMessage { + get { return failureMessage_; } + set { + failureMessage_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as TestStatus); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(TestStatus other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Name != other.Name) return false; + if (FailureMessage != other.FailureMessage) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (Name.Length != 0) hash ^= Name.GetHashCode(); + if (FailureMessage.Length != 0) hash ^= FailureMessage.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (FailureMessage.Length != 0) { + output.WriteRawTag(18); + output.WriteString(FailureMessage); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (FailureMessage.Length != 0) { + output.WriteRawTag(18); + output.WriteString(FailureMessage); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (Name.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Name); + } + if (FailureMessage.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(FailureMessage); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(TestStatus other) { + if (other == null) { + return; + } + if (other.Name.Length != 0) { + Name = other.Name; + } + if (other.FailureMessage.Length != 0) { + FailureMessage = other.FailureMessage; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + FailureMessage = input.ReadString(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + FailureMessage = input.ReadString(); + break; + } + } + } + } + #endif + + } + /// /// The conformance runner will request a list of failures as the first request. /// This will be known by message_type == "conformance.FailureSet", a conformance @@ -124,7 +367,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[0]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[1]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -144,7 +387,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public FailureSet(FailureSet other) : this() { - failure_ = other.failure_.Clone(); + test_ = other.test_.Clone(); _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -154,15 +397,15 @@ namespace Conformance { return new FailureSet(this); } - /// Field number for the "failure" field. - public const int FailureFieldNumber = 1; - private static readonly pb::FieldCodec _repeated_failure_codec - = pb::FieldCodec.ForString(10); - private readonly pbc::RepeatedField failure_ = new pbc::RepeatedField(); + /// Field number for the "test" field. + public const int TestFieldNumber = 2; + private static readonly pb::FieldCodec _repeated_test_codec + = pb::FieldCodec.ForMessage(18, global::Conformance.TestStatus.Parser); + private readonly pbc::RepeatedField test_ = new pbc::RepeatedField(); [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public pbc::RepeatedField Failure { - get { return failure_; } + public pbc::RepeatedField Test { + get { return test_; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -180,7 +423,7 @@ namespace Conformance { if (ReferenceEquals(other, this)) { return true; } - if(!failure_.Equals(other.failure_)) return false; + if(!test_.Equals(other.test_)) return false; return Equals(_unknownFields, other._unknownFields); } @@ -188,7 +431,7 @@ namespace Conformance { [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override int GetHashCode() { int hash = 1; - hash ^= failure_.GetHashCode(); + hash ^= test_.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -207,7 +450,7 @@ namespace Conformance { #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE output.WriteRawMessage(this); #else - failure_.WriteTo(output, _repeated_failure_codec); + test_.WriteTo(output, _repeated_test_codec); if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -218,7 +461,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { - failure_.WriteTo(ref output, _repeated_failure_codec); + test_.WriteTo(ref output, _repeated_test_codec); if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -229,7 +472,7 @@ namespace Conformance { [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public int CalculateSize() { int size = 0; - size += failure_.CalculateSize(_repeated_failure_codec); + size += test_.CalculateSize(_repeated_test_codec); if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -242,7 +485,7 @@ namespace Conformance { if (other == null) { return; } - failure_.Add(other.failure_); + test_.Add(other.test_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -262,8 +505,8 @@ namespace Conformance { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; - case 10: { - failure_.AddEntriesFrom(input, _repeated_failure_codec); + case 18: { + test_.AddEntriesFrom(input, _repeated_test_codec); break; } } @@ -285,8 +528,8 @@ namespace Conformance { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; - case 10: { - failure_.AddEntriesFrom(ref input, _repeated_failure_codec); + case 18: { + test_.AddEntriesFrom(ref input, _repeated_test_codec); break; } } @@ -318,7 +561,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[1]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[2]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -945,7 +1188,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[2]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[3]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -1665,7 +1908,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[3]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[4]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] From be3d8cb7dcef37d81ab146326abb07353b728742 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Jul 2024 06:05:04 -0700 Subject: [PATCH 058/107] Bring in {UpbStrToStringView, UpbStrFromStringView}, MessageAlloc/Decode/Encode, and some internal template helpers into hpb PiperOrigin-RevId: 655136424 --- hpb/hpb.cc | 4 +- hpb/hpb.h | 123 ++++++++++++++------------- hpb_generator/gen_accessors.cc | 9 +- hpb_generator/gen_repeated_fields.cc | 6 +- 4 files changed, 75 insertions(+), 67 deletions(-) diff --git a/hpb/hpb.cc b/hpb/hpb.cc index d63c7d98d9..3437a5fb49 100644 --- a/hpb/hpb.cc +++ b/hpb/hpb.cc @@ -27,7 +27,7 @@ #include "upb/wire/decode.h" #include "upb/wire/encode.h" -namespace protos { +namespace hpb { // begin:google_only absl::Status MessageAllocationError(SourceLocation loc) { @@ -80,7 +80,9 @@ absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc // return absl::Status(absl::StatusCode::kUnknown, "Upb message parse error"); // } // end:github_only +} // namespace hpb +namespace protos { namespace internal { using ::hpb::internal::upb_extension_locker_global; diff --git a/hpb/hpb.h b/hpb/hpb.h index 171da7896e..73221b05d3 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -82,13 +82,7 @@ class Ptr final { template Ptr(T* m) -> Ptr; -} // namespace hpb - -namespace protos { -using hpb::Arena; -using hpb::Ptr; -class ExtensionRegistry; - +// TODO: b/354766950 - Move upb-specific chunks out of hpb header inline absl::string_view UpbStrToStringView(upb_StringView str) { return absl::string_view(str.data, str.size); } @@ -102,12 +96,6 @@ inline upb_StringView UpbStrFromStringView(absl::string_view str, return upb_StringView_FromDataAndSize(buffer, str_size); } -template -typename T::Proxy CreateMessage(::hpb::Arena& arena) { - return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), - arena.ptr()); -} - // begin:github_only // // This type exists to work around an absl type that has not yet been // // released. @@ -134,6 +122,48 @@ absl::Status MessageDecodeError(upb_DecodeStatus status, absl::Status MessageEncodeError(upb_EncodeStatus status, SourceLocation loc = SourceLocation::current()); +namespace internal { +template +struct RemovePtr; + +template +struct RemovePtr> { + using type = T; +}; + +template +struct RemovePtr { + using type = T; +}; + +template +using RemovePtrT = typename RemovePtr::type; + +template , + typename = std::enable_if_t>> +using PtrOrRaw = T; + +template +using EnableIfHpbClass = std::enable_if_t< + std::is_base_of::value && + std::is_base_of::value>; + +template +using EnableIfMutableProto = std::enable_if_t::value>; +} // namespace internal + +} // namespace hpb + +namespace protos { +using hpb::Arena; +using hpb::ExtensionNotFoundError; +using hpb::MessageAllocationError; +using hpb::MessageDecodeError; +using hpb::MessageEncodeError; +using hpb::Ptr; +using hpb::SourceLocation; +class ExtensionRegistry; + namespace internal { struct PrivateAccess { template @@ -261,36 +291,13 @@ absl::Status MoveExtension(upb_Message* message, upb_Arena* message_arena, absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, const upb_MiniTableExtension* ext, const upb_Message* extension); +} // namespace internal template -struct RemovePtr; - -template -struct RemovePtr> { - using type = T; -}; - -template -struct RemovePtr { - using type = T; -}; - -template -using RemovePtrT = typename RemovePtr::type; - -template , - typename = std::enable_if_t>> -using PtrOrRaw = T; - -template -using EnableIfHpbClass = std::enable_if_t< - std::is_base_of::value && - std::is_base_of::value>; - -template -using EnableIfMutableProto = std::enable_if_t::value>; - -} // namespace internal +typename T::Proxy CreateMessage(::hpb::Arena& arena) { + return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), + arena.ptr()); +} template void DeepCopy(Ptr source_message, Ptr target_message) { @@ -328,7 +335,7 @@ void DeepCopy(const T* source_message, T* target_message) { } template -void ClearMessage(internal::PtrOrRaw message) { +void ClearMessage(hpb::internal::PtrOrRaw message) { auto ptr = Ptr(message); auto minitable = internal::GetMiniTable(ptr); upb_Message_Clear(internal::GetInternalMsg(ptr), minitable); @@ -360,7 +367,7 @@ class ExtensionRegistry { }; template > + typename = hpb::internal::EnableIfHpbClass> ABSL_MUST_USE_RESULT bool HasExtension( Ptr message, const ::protos::internal::ExtensionIdentifier& id) { @@ -369,7 +376,7 @@ ABSL_MUST_USE_RESULT bool HasExtension( } template > + typename = hpb::internal::EnableIfHpbClass> ABSL_MUST_USE_RESULT bool HasExtension( const T* message, const ::protos::internal::ExtensionIdentifier& id) { @@ -377,8 +384,8 @@ ABSL_MUST_USE_RESULT bool HasExtension( } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> void ClearExtension( Ptr message, const ::protos::internal::ExtensionIdentifier& id) { @@ -388,7 +395,7 @@ void ClearExtension( } template > + typename = hpb::internal::EnableIfHpbClass> void ClearExtension( T* message, const ::protos::internal::ExtensionIdentifier& id) { @@ -396,8 +403,8 @@ void ClearExtension( } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> absl::Status SetExtension( Ptr message, const ::protos::internal::ExtensionIdentifier& id, @@ -410,8 +417,8 @@ absl::Status SetExtension( } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> absl::Status SetExtension( Ptr message, const ::protos::internal::ExtensionIdentifier& id, @@ -424,8 +431,8 @@ absl::Status SetExtension( } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> absl::Status SetExtension( Ptr message, const ::protos::internal::ExtensionIdentifier& id, @@ -440,7 +447,7 @@ absl::Status SetExtension( } template > + typename = hpb::internal::EnableIfHpbClass> absl::Status SetExtension( T* message, const ::protos::internal::ExtensionIdentifier& id, const Extension& value) { @@ -448,7 +455,7 @@ absl::Status SetExtension( } template > + typename = hpb::internal::EnableIfHpbClass> absl::Status SetExtension( T* message, const ::protos::internal::ExtensionIdentifier& id, Extension&& value) { @@ -457,7 +464,7 @@ absl::Status SetExtension( } template > + typename = hpb::internal::EnableIfHpbClass> absl::Status SetExtension( T* message, const ::protos::internal::ExtensionIdentifier& id, Ptr value) { @@ -465,7 +472,7 @@ absl::Status SetExtension( } template > + typename = hpb::internal::EnableIfHpbClass> absl::StatusOr> GetExtension( Ptr message, const ::protos::internal::ExtensionIdentifier& id) { @@ -483,7 +490,7 @@ absl::StatusOr> GetExtension( } template > + typename = hpb::internal::EnableIfHpbClass> absl::StatusOr> GetExtension( const T* message, const ::protos::internal::ExtensionIdentifier& id) { diff --git a/hpb_generator/gen_accessors.cc b/hpb_generator/gen_accessors.cc index 048efcd3df..d6eb5dac98 100644 --- a/hpb_generator/gen_accessors.cc +++ b/hpb_generator/gen_accessors.cc @@ -222,7 +222,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { output( R"cc( $1 $0::$2() const { - return ::protos::UpbStrToStringView($3_$4(msg_)); + return ::hpb::UpbStrToStringView($3_$4(msg_)); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -231,7 +231,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { output( R"cc( void $0::set_$2($1 value) { - $4_set_$3(msg_, ::protos::UpbStrFromStringView(value, $5)); + $4_set_$3(msg_, ::hpb::UpbStrFromStringView(value, $5)); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -347,9 +347,8 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, output( R"cc( bool $0::set_$1($2 key, $3 value) { - $5return $4_$7_set(msg_, $6, - ::protos::UpbStrFromStringView(value, arena_), - arena_); + $5return $4_$7_set( + msg_, $6, ::hpb::UpbStrFromStringView(value, arena_), arena_); } )cc", class_name, resolved_field_name, CppConstType(key), CppConstType(val), diff --git a/hpb_generator/gen_repeated_fields.cc b/hpb_generator/gen_repeated_fields.cc index 163648e3a8..3ea3014ddb 100644 --- a/hpb_generator/gen_repeated_fields.cc +++ b/hpb_generator/gen_repeated_fields.cc @@ -205,7 +205,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_mutable_$4(msg_, &len); assert(index < len); - return ::protos::UpbStrToStringView(*(ptr + index)); + return ::hpb::UpbStrToStringView(*(ptr + index)); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -220,7 +220,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, output( R"cc( bool $0::add_$2($1 val) { - return $3_add_$4(msg_, ::protos::UpbStrFromStringView(val, arena_), arena_); + return $3_add_$4(msg_, ::hpb::UpbStrFromStringView(val, arena_), arena_); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -231,7 +231,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_mutable_$4(msg_, &len); assert(index < len); - *(ptr + index) = ::protos::UpbStrFromStringView(val, arena_); + *(ptr + index) = ::hpb::UpbStrFromStringView(val, arena_); } )cc", class_name, CppConstType(field), resolved_field_name, From 3c95fc8b762d96a18effb508b46cee1e4d8c2946 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Jul 2024 07:54:29 -0700 Subject: [PATCH 059/107] Create AsView/IntoView/AsMut/IntoMut traits These 'verb' traits are more in line with our plans for the common message operations. PiperOrigin-RevId: 655163895 --- rust/codegen_traits.rs | 2 +- rust/map.rs | 35 +++-- rust/primitive.rs | 21 ++- rust/proxied.rs | 131 ++++++++++++------- rust/repeated.rs | 42 ++++-- rust/shared.rs | 3 +- rust/string.rs | 17 ++- rust/test/shared/accessors_repeated_test.rs | 2 +- src/google/protobuf/compiler/rust/enum.cc | 11 +- src/google/protobuf/compiler/rust/message.cc | 55 +++++--- 10 files changed, 216 insertions(+), 103 deletions(-) diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index fe5de9916b..c48c5c02a1 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -43,7 +43,7 @@ pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> /// A trait that all generated message muts implement. pub trait MessageMut<'msg>: - MutProxy<'msg, Proxied = Self::Message> + MutProxy<'msg, MutProxied = Self::Message> // Read traits: + Debug + Serialize // Write traits: diff --git a/rust/map.rs b/rust/map.rs index 0e7953f872..7013226ec7 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -6,7 +6,8 @@ // https://developers.google.com/open-source/licenses/bsd use crate::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, + AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, View, + ViewProxy, __internal::Private, __runtime::{InnerMap, InnerMapMut, RawMap, RawMapIter}, }; @@ -112,14 +113,18 @@ impl> MutProxied for Map { type Mut<'msg> = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } -impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, V> { +impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapView<'msg, K, V> { type Proxied = Map; - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> MapView<'_, K, V> { *self } +} - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> +impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoView<'msg> for MapView<'msg, K, V> { + fn into_view<'shorter>(self) -> MapView<'shorter, K, V> where 'msg: 'shorter, { @@ -129,14 +134,18 @@ impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, impl<'msg, K: Proxied, V: ProxiedInMapValue> ViewProxy<'msg> for MapView<'msg, K, V> {} -impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapMut<'msg, K, V> { +impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapMut<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapMut<'msg, K, V> { type Proxied = Map; - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> MapView<'_, K, V> { MapView { raw: self.inner.raw, _phantom: PhantomData } } +} - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> +impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoView<'msg> for MapMut<'msg, K, V> { + fn into_view<'shorter>(self) -> MapView<'shorter, K, V> where 'msg: 'shorter, { @@ -144,12 +153,16 @@ impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapMut<'msg, K, } } -impl<'msg, K: Proxied, V: ProxiedInMapValue> MutProxy<'msg> for MapMut<'msg, K, V> { - fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { +impl<'msg, K: Proxied, V: ProxiedInMapValue> AsMut for MapMut<'msg, K, V> { + type MutProxied = Map; + + fn as_mut(&mut self) -> MapMut<'_, K, V> { MapMut { inner: self.inner, _phantom: PhantomData } } +} - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> +impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoMut<'msg> for MapMut<'msg, K, V> { + fn into_mut<'shorter>(self) -> MapMut<'shorter, K, V> where 'msg: 'shorter, { @@ -157,6 +170,8 @@ impl<'msg, K: Proxied, V: ProxiedInMapValue> MutProxy<'msg> for MapMut<'msg, } } +impl<'msg, K: Proxied, V: ProxiedInMapValue> MutProxy<'msg> for MapMut<'msg, K, V> {} + impl Map where K: Proxied, diff --git a/rust/primitive.rs b/rust/primitive.rs index ab50352f14..06c90f1536 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd use crate::__internal::Private; -use crate::{IntoProxied, Proxied, Proxy, View, ViewProxy}; +use crate::{AsView, IntoProxied, IntoView, Proxied, Proxy, ViewProxy}; macro_rules! impl_singular_primitives { ($($t:ty),*) => { @@ -15,15 +15,22 @@ macro_rules! impl_singular_primitives { } impl<'msg> Proxy<'msg> for $t { + } + + impl AsView for $t { type Proxied = $t; - fn as_view(&self) -> View<'_, Self::Proxied> { - *self - } + fn as_view(&self) -> $t { + *self + } + } - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> { - self - } + impl<'msg> IntoView<'msg> for $t { + fn into_view<'shorter>(self) -> $t + where + 'msg: 'shorter { + self + } } impl<'msg> ViewProxy<'msg> for $t {} diff --git a/rust/proxied.rs b/rust/proxied.rs index f252b2e879..3092109566 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -72,7 +72,7 @@ pub trait MutProxied: Proxied { /// `&'msg mut T`. /// /// Most code should use the type alias [`Mut`]. - type Mut<'msg>: MutProxy<'msg, Proxied = Self> + type Mut<'msg>: MutProxy<'msg, MutProxied = Self> where Self: 'msg; } @@ -90,13 +90,13 @@ pub type View<'msg, T> = ::View<'msg>; #[allow(dead_code)] pub type Mut<'msg, T> = ::Mut<'msg>; -/// Declares conversion operations common to all proxies (both views and mut -/// proxies). +/// Used to semantically do a cheap "to-reference" conversion. This is +/// implemented on both owned `Proxied` types as well as ViewProxy and MutProxy +/// types. /// -/// This trait is intentionally made non-object-safe to prevent a potential -/// future incompatible change. -pub trait Proxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { - type Proxied: 'msg + Proxied; +/// On ViewProxy this will behave as a reborrow into a shorter lifetime. +pub trait AsView { + type Proxied: Proxied; /// Converts a borrow into a `View` with the lifetime of that borrow. /// @@ -118,7 +118,16 @@ pub trait Proxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { /// /// [invariant]: https://doc.rust-lang.org/nomicon/subtyping.html#variance fn as_view(&self) -> View<'_, Self::Proxied>; +} +/// Used to turn another 'borrow' into a ViewProxy. +/// +/// On a MutProxy this borrows to a View (semantically matching turning a `&mut +/// T` into a `&T`). +/// +/// On a ViewProxy this will behave as a reborrow into a shorter lifetime +/// (semantically matching a `&'a T` into a `&'b T` where `'a: 'b`). +pub trait IntoView<'msg>: AsView { /// Converts into a `View` with a potentially shorter lifetime. /// /// In non-generic code we don't need to use `into_view` because the proxy @@ -149,41 +158,22 @@ pub trait Proxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { 'msg: 'shorter; } -/// Declares conversion operations common to view proxies. -pub trait ViewProxy<'msg>: Proxy<'msg> + Copy + Send {} - -/// Declares operations common to all mut proxies. +/// Used to semantically do a cheap "to-mut-reference" conversion. This is +/// implemented on both owned `Proxied` types as well as MutProxy types. /// -/// This trait is intentionally made non-object-safe to prevent a potential -/// future incompatible change. -pub trait MutProxy<'msg>: Proxy<'msg> -where - Self::Proxied: MutProxied, -{ - /// Gets an immutable view of this field. This is shorthand for `as_view`. - /// - /// This provides a shorter lifetime than `into_view` but can also be called - /// multiple times - if the result of `get` is not living long enough - /// for your use, use that instead. - fn get(&self) -> View<'_, Self::Proxied> { - self.as_view() - } +/// On MutProxy this will behave as a reborrow into a shorter lifetime. +pub trait AsMut: AsView { + type MutProxied: MutProxied; /// Converts a borrow into a `Mut` with the lifetime of that borrow. - /// - /// This function enables calling multiple methods consuming `self`, for - /// example: - /// - /// ```ignore - /// let mut sub: Mut = msg.submsg_mut(); - /// sub.as_mut().field_x_mut().set(10); // field_x_mut is fn(self) - /// sub.field_y_mut().set(20); // `sub` is now consumed - /// ``` - /// - /// `as_mut` is also useful in generic code to explicitly perform the - /// operation that in concrete code coercion would perform implicitly. - fn as_mut(&mut self) -> Mut<'_, Self::Proxied>; + fn as_mut(&mut self) -> Mut<'_, Self::MutProxied>; +} +/// Used to turn another 'borrow' into a MutProxy. +/// +/// On a MutProxy this will behave as a reborrow into a shorter lifetime +/// (semantically matching a `&mut 'a T` into a `&mut 'b T` where `'a: 'b`). +pub trait IntoMut<'msg>: AsMut { /// Converts into a `Mut` with a potentially shorter lifetime. /// /// In non-generic code we don't need to use `into_mut` because the proxy @@ -206,11 +196,36 @@ where /// ``` /// /// [invariant]: https://doc.rust-lang.org/nomicon/subtyping.html#variance - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> + fn into_mut<'shorter>(self) -> Mut<'shorter, Self::MutProxied> where 'msg: 'shorter; } +/// Declares conversion operations common to all proxies (both views and mut +/// proxies). +/// +/// This trait is intentionally made non-object-safe to prevent a potential +/// future incompatible change. +pub trait Proxy<'msg>: 'msg + IntoView<'msg> + Sync + Unpin + Sized + Debug {} + +/// Declares conversion operations common to view proxies. +pub trait ViewProxy<'msg>: Proxy<'msg> + Copy + Send {} + +/// Declares operations common to all mut proxies. +/// +/// This trait is intentionally made non-object-safe to prevent a potential +/// future incompatible change. +pub trait MutProxy<'msg>: Proxy<'msg> + AsMut + IntoMut<'msg> { + /// Gets an immutable view of this field. This is shorthand for `as_view`. + /// + /// This provides a shorter lifetime than `into_view` but can also be called + /// multiple times - if the result of `get` is not living long enough + /// for your use, use that instead. + fn get(&self) -> View<'_, Self::Proxied> { + self.as_view() + } +} + /// A value to `Proxied`-value conversion that consumes the input value. /// /// All setter functions accept types that implement `IntoProxied`. The purpose @@ -263,14 +278,20 @@ mod tests { } } - impl<'msg> Proxy<'msg> for MyProxiedView<'msg> { + impl<'msg> Proxy<'msg> for MyProxiedView<'msg> {} + + impl<'msg> ViewProxy<'msg> for MyProxiedView<'msg> {} + + impl<'msg> AsView for MyProxiedView<'msg> { type Proxied = MyProxied; - fn as_view(&self) -> View<'msg, MyProxied> { + fn as_view(&self) -> MyProxiedView<'msg> { *self } + } - fn into_view<'shorter>(self) -> View<'shorter, MyProxied> + impl<'msg> IntoView<'msg> for MyProxiedView<'msg> { + fn into_view<'shorter>(self) -> MyProxiedView<'shorter> where 'msg: 'shorter, { @@ -278,19 +299,22 @@ mod tests { } } - impl<'msg> ViewProxy<'msg> for MyProxiedView<'msg> {} - #[derive(Debug)] struct MyProxiedMut<'msg> { my_proxied_ref: &'msg mut MyProxied, } - impl<'msg> Proxy<'msg> for MyProxiedMut<'msg> { + impl<'msg> Proxy<'msg> for MyProxiedMut<'msg> {} + + impl<'msg> AsView for MyProxiedMut<'msg> { type Proxied = MyProxied; - fn as_view(&self) -> View<'_, MyProxied> { + fn as_view(&self) -> MyProxiedView<'_> { MyProxiedView { my_proxied_ref: self.my_proxied_ref } } + } + + impl<'msg> IntoView<'msg> for MyProxiedMut<'msg> { fn into_view<'shorter>(self) -> View<'shorter, MyProxied> where 'msg: 'shorter, @@ -299,12 +323,16 @@ mod tests { } } - impl<'msg> MutProxy<'msg> for MyProxiedMut<'msg> { - fn as_mut(&mut self) -> Mut<'_, MyProxied> { + impl<'msg> AsMut for MyProxiedMut<'msg> { + type MutProxied = MyProxied; + + fn as_mut(&mut self) -> MyProxiedMut<'_> { MyProxiedMut { my_proxied_ref: self.my_proxied_ref } } + } - fn into_mut<'shorter>(self) -> Mut<'shorter, MyProxied> + impl<'msg> IntoMut<'msg> for MyProxiedMut<'msg> { + fn into_mut<'shorter>(self) -> MyProxiedMut<'shorter> where 'msg: 'shorter, { @@ -312,6 +340,8 @@ mod tests { } } + impl<'msg> MutProxy<'msg> for MyProxiedMut<'msg> {} + #[googletest::test] fn test_as_view() { let my_proxied = MyProxied { val: "Hello World".to_string() }; @@ -427,7 +457,8 @@ mod tests { // `[x, y]` fails to compile because `'a` is not the same as `'b` and the `Mut` // lifetime parameter is (conservatively) invariant. // `[x.as_mut(), y]` fails because that borrow cannot outlive `'b`. - [x.into_mut(), y] + let tmp: Mut<'b, T> = x.into_mut(); + [tmp, y] } #[googletest::test] diff --git a/rust/repeated.rs b/rust/repeated.rs index d7a02e4c5d..0db3f64a96 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -15,7 +15,8 @@ use std::iter::FusedIterator; use std::marker::PhantomData; use crate::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, + AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, View, + ViewProxy, __internal::Private, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -397,17 +398,24 @@ where type Mut<'msg> = RepeatedMut<'msg, T> where Repeated: 'msg; } -impl<'msg, T> Proxy<'msg> for RepeatedView<'msg, T> +impl<'msg, T> Proxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> AsView for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg, { type Proxied = Repeated; #[inline] - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> View<'msg, Self::Proxied> { *self } +} +impl<'msg, T> IntoView<'msg> for RepeatedView<'msg, T> +where + T: ProxiedInRepeated + 'msg, +{ #[inline] fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> where @@ -419,19 +427,26 @@ where impl<'msg, T> ViewProxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} -impl<'msg, T> Proxy<'msg> for RepeatedMut<'msg, T> +impl<'msg, T> Proxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> AsView for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg, { type Proxied = Repeated; #[inline] - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> RepeatedView<'_, T> { RepeatedView { raw: self.inner.raw, _phantom: PhantomData } } +} +impl<'msg, T> IntoView<'msg> for RepeatedMut<'msg, T> +where + T: ProxiedInRepeated + 'msg, +{ #[inline] - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> + fn into_view<'shorter>(self) -> RepeatedView<'shorter, T> where 'msg: 'shorter, { @@ -439,17 +454,24 @@ where } } -impl<'msg, T> MutProxy<'msg> for RepeatedMut<'msg, T> +impl<'msg, T> AsMut for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg, { + type MutProxied = Repeated; + #[inline] - fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { + fn as_mut(&mut self) -> RepeatedMut<'_, T> { RepeatedMut { inner: self.inner, _phantom: PhantomData } } +} +impl<'msg, T> IntoMut<'msg> for RepeatedMut<'msg, T> +where + T: ProxiedInRepeated + 'msg, +{ #[inline] - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> + fn into_mut<'shorter>(self) -> RepeatedMut<'shorter, T> where 'msg: 'shorter, { @@ -457,6 +479,8 @@ where } } +impl<'msg, T> MutProxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} + impl<'msg, T> iter::Iterator for RepeatedIter<'msg, T> where T: ProxiedInRepeated + 'msg, diff --git a/rust/shared.rs b/rust/shared.rs index 113292c875..716f34262f 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -33,7 +33,8 @@ pub mod __public { pub use crate::optional::Optional; pub use crate::proto; pub use crate::proxied::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, + AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, + View, ViewProxy, }; pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, diff --git a/rust/string.rs b/rust/string.rs index e3b8365b40..3a90ba3db8 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -11,7 +11,10 @@ use crate::__internal::Private; use crate::__runtime::{InnerProtoString, PtrAndLen, RawMessage}; -use crate::{IntoProxied, Mut, MutProxied, MutProxy, Optional, Proxied, Proxy, View, ViewProxy}; +use crate::{ + AsView, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Optional, Proxied, Proxy, View, + ViewProxy, +}; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use std::convert::{AsMut, AsRef}; @@ -114,13 +117,17 @@ impl IntoProxied for Arc<[u8]> { } } -impl<'msg> Proxy<'msg> for &'msg [u8] { +impl<'msg> Proxy<'msg> for &'msg [u8] {} + +impl AsView for &[u8] { type Proxied = ProtoBytes; fn as_view(&self) -> &[u8] { self } +} +impl<'msg> IntoView<'msg> for &'msg [u8] { fn into_view<'shorter>(self) -> &'shorter [u8] where 'msg: 'shorter, @@ -481,13 +488,17 @@ impl Proxied for ProtoString { type View<'msg> = &'msg ProtoStr; } -impl<'msg> Proxy<'msg> for &'msg ProtoStr { +impl<'msg> Proxy<'msg> for &'msg ProtoStr {} + +impl AsView for &ProtoStr { type Proxied = ProtoString; fn as_view(&self) -> &ProtoStr { self } +} +impl<'msg> IntoView<'msg> for &'msg ProtoStr { fn into_view<'shorter>(self) -> &'shorter ProtoStr where 'msg: 'shorter, diff --git a/rust/test/shared/accessors_repeated_test.rs b/rust/test/shared/accessors_repeated_test.rs index 8898d9c096..c1de003561 100644 --- a/rust/test/shared/accessors_repeated_test.rs +++ b/rust/test/shared/accessors_repeated_test.rs @@ -7,7 +7,7 @@ use googletest::prelude::*; use paste::paste; -use protobuf::Proxy; +use protobuf::AsView; use unittest_rust_proto::{test_all_types, test_all_types::NestedMessage, TestAllTypes}; macro_rules! generate_repeated_numeric_test { diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index a2b1143146..9513c93eb2 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -405,20 +405,23 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { type View<'a> = $name$; } - impl $pb$::Proxy<'_> for $name$ { + impl $pb$::Proxy<'_> for $name$ {} + impl $pb$::ViewProxy<'_> for $name$ {} + + impl $pb$::AsView for $name$ { type Proxied = $name$; fn as_view(&self) -> $name$ { *self } + } - fn into_view<'shorter>(self) -> $pb$::View<'shorter, $name$> { + impl<'msg> $pb$::IntoView<'msg> for $name$ { + fn into_view<'shorter>(self) -> $name$ where 'msg: 'shorter { self } } - impl $pb$::ViewProxy<'_> for $name$ {} - unsafe impl $pb$::ProxiedInRepeated for $name$ { fn repeated_new(_private: $pbi$::Private) -> $pb$::Repeated { $pbr$::new_enum_repeated($pbi$::Private) diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index d8b354d6ac..b0a16df41c 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -260,7 +260,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - $pb$::IntoProxied::into_proxied($pb$::Proxy::into_view(self), _private) + $pb$::IntoProxied::into_proxied($pb$::IntoView::into_view(self), _private) } } @@ -289,7 +289,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - $pb$::IntoProxied::into_proxied($pb$::Proxy::into_view(self), _private) + $pb$::IntoProxied::into_proxied($pb$::IntoView::into_view(self), _private) } } @@ -1055,19 +1055,24 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { // - `$Msg$View` does not use thread-local data. unsafe impl Send for $Msg$View<'_> {} - impl<'msg> $pb$::Proxy<'msg> for $Msg$View<'msg> { - type Proxied = $Msg$; + impl<'msg> $pb$::Proxy<'msg> for $Msg$View<'msg> {} + impl<'msg> $pb$::ViewProxy<'msg> for $Msg$View<'msg> {} + impl<'msg> $pb$::AsView for $Msg$View<'msg> { + type Proxied = $Msg$; fn as_view(&self) -> $pb$::View<'msg, $Msg$> { *self } - fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> where 'msg: 'shorter { + } + + impl<'msg> $pb$::IntoView<'msg> for $Msg$View<'msg> { + fn into_view<'shorter>(self) -> $Msg$View<'shorter> + where + 'msg: 'shorter { self } } - impl<'msg> $pb$::ViewProxy<'msg> for $Msg$View<'msg> {} - $into_proxied_impl$ $repeated_impl$ @@ -1124,11 +1129,11 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } pub fn serialize(&self) -> Result, $pb$::SerializeError> { - $pb$::Proxy::as_view(self).serialize() + $pb$::AsView::as_view(self).serialize() } pub fn to_owned(&self) -> $Msg$ { - $pb$::Proxy::as_view(self).to_owned() + $pb$::AsView::as_view(self).to_owned() } $msg_merge_from$ @@ -1146,23 +1151,39 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { // splitting, synchronous access of an arena is impossible. unsafe impl Sync for $Msg$Mut<'_> {} - impl<'msg> $pb$::MutProxy<'msg> for $Msg$Mut<'msg> { - fn as_mut(&mut self) -> $pb$::Mut<'_, $Msg$> { - $Msg$Mut { inner: self.inner } - } - fn into_mut<'shorter>(self) -> $pb$::Mut<'shorter, $Msg$> where 'msg : 'shorter { self } - } + impl<'msg> $pb$::Proxy<'msg> for $Msg$Mut<'msg> {} + impl<'msg> $pb$::MutProxy<'msg> for $Msg$Mut<'msg> {} - impl<'msg> $pb$::Proxy<'msg> for $Msg$Mut<'msg> { + impl<'msg> $pb$::AsView for $Msg$Mut<'msg> { type Proxied = $Msg$; fn as_view(&self) -> $pb$::View<'_, $Msg$> { $Msg$View { msg: self.raw_msg(), _phantom: $std$::marker::PhantomData } } - fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> where 'msg: 'shorter { + } + + impl<'msg> $pb$::IntoView<'msg> for $Msg$Mut<'msg> { + fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> + where + 'msg: 'shorter { $Msg$View { msg: self.raw_msg(), _phantom: $std$::marker::PhantomData } } } + impl<'msg> $pb$::AsMut for $Msg$Mut<'msg> { + type MutProxied = $Msg$; + fn as_mut(&mut self) -> $Msg$Mut<'msg> { + $Msg$Mut { inner: self.inner } + } + } + + impl<'msg> $pb$::IntoMut<'msg> for $Msg$Mut<'msg> { + fn into_mut<'shorter>(self) -> $Msg$Mut<'shorter> + where + 'msg: 'shorter { + self + } + } + #[allow(dead_code)] impl $Msg$ { pub fn new() -> Self { From a0387c19237b89ace95a353e7d76ea38fb05c6d5 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Jul 2024 08:33:57 -0700 Subject: [PATCH 060/107] Remove protos.cc - abrogated by hpb.cc PiperOrigin-RevId: 655176487 --- protos/protos.cc | 185 ----------------------------------------------- 1 file changed, 185 deletions(-) delete mode 100644 protos/protos.cc diff --git a/protos/protos.cc b/protos/protos.cc deleted file mode 100644 index fedbb6bc30..0000000000 --- a/protos/protos.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#include "protos/protos.h" - -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" -#include "protos/protos_extension_lock.h" -#include "upb/mem/arena.h" -#include "upb/message/accessors.h" -#include "upb/message/copy.h" -#include "upb/message/message.h" -#include "upb/message/promote.h" -#include "upb/message/value.h" -#include "upb/mini_table/extension.h" -#include "upb/mini_table/extension_registry.h" -#include "upb/mini_table/message.h" -#include "upb/wire/decode.h" -#include "upb/wire/encode.h" - -namespace protos { - -// begin:google_only -// absl::Status MessageAllocationError(SourceLocation loc) { -// return absl::Status(absl::StatusCode::kInternal, -// "Upb message allocation error", loc); -// } -// -// absl::Status ExtensionNotFoundError(int extension_number, SourceLocation loc) { -// return absl::Status( -// absl::StatusCode::kInternal, -// absl::StrFormat("Extension %d not found", extension_number), loc); -// } -// -// absl::Status MessageEncodeError(upb_EncodeStatus status, SourceLocation loc) { -// return absl::Status(absl::StatusCode::kInternal, -// absl::StrFormat("Upb message encoding error %d", status), -// loc -// -// ); -// } -// -// absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc -// -// ) { -// return absl::Status(absl::StatusCode::kInternal, -// absl::StrFormat("Upb message parse error %d", status), loc -// -// ); -// } -// end:google_only - -// begin:github_only -absl::Status MessageAllocationError(SourceLocation loc) { - return absl::Status(absl::StatusCode::kUnknown, - "Upb message allocation error"); -} - -absl::Status ExtensionNotFoundError(int ext_number, SourceLocation loc) { - return absl::Status(absl::StatusCode::kUnknown, - absl::StrFormat("Extension %d not found", ext_number)); -} - -absl::Status MessageEncodeError(upb_EncodeStatus s, SourceLocation loc) { - return absl::Status(absl::StatusCode::kUnknown, "Encoding error"); -} - -absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc - -) { - return absl::Status(absl::StatusCode::kUnknown, "Upb message parse error"); -} -// end:github_only - -namespace internal { - -upb_ExtensionRegistry* GetUpbExtensions( - const ExtensionRegistry& extension_registry) { - return extension_registry.registry_; -} - -/** - * MessageLock(msg) acquires lock on msg when constructed and releases it when - * destroyed. - */ -class MessageLock { - public: - explicit MessageLock(const upb_Message* msg) : msg_(msg) { - UpbExtensionLocker locker = - upb_extension_locker_global.load(std::memory_order_acquire); - unlocker_ = (locker != nullptr) ? locker(msg) : nullptr; - } - MessageLock(const MessageLock&) = delete; - void operator=(const MessageLock&) = delete; - ~MessageLock() { - if (unlocker_ != nullptr) { - unlocker_(msg_); - } - } - - private: - const upb_Message* msg_; - UpbExtensionUnlocker unlocker_; -}; - -bool HasExtensionOrUnknown(const upb_Message* msg, - const upb_MiniTableExtension* eid) { - MessageLock msg_lock(msg); - if (upb_Message_HasExtension(msg, eid)) return true; - - const int number = upb_MiniTableExtension_Number(eid); - return upb_Message_FindUnknown(msg, number, 0).status == kUpb_FindUnknown_Ok; -} - -bool GetOrPromoteExtension(upb_Message* msg, const upb_MiniTableExtension* eid, - upb_Arena* arena, upb_MessageValue* value) { - MessageLock msg_lock(msg); - upb_GetExtension_Status ext_status = upb_Message_GetOrPromoteExtension( - (upb_Message*)msg, eid, 0, arena, value); - return ext_status == kUpb_GetExtension_Ok; -} - -absl::StatusOr Serialize(const upb_Message* message, - const upb_MiniTable* mini_table, - upb_Arena* arena, int options) { - MessageLock msg_lock(message); - size_t len; - char* ptr; - upb_EncodeStatus status = - upb_Encode(message, mini_table, options, arena, &ptr, &len); - if (status == kUpb_EncodeStatus_Ok) { - return absl::string_view(ptr, len); - } - return MessageEncodeError(status); -} - -void DeepCopy(upb_Message* target, const upb_Message* source, - const upb_MiniTable* mini_table, upb_Arena* arena) { - MessageLock msg_lock(source); - upb_Message_DeepCopy(target, source, mini_table, arena); -} - -upb_Message* DeepClone(const upb_Message* source, - const upb_MiniTable* mini_table, upb_Arena* arena) { - MessageLock msg_lock(source); - return upb_Message_DeepClone(source, mini_table, arena); -} - -absl::Status MoveExtension(upb_Message* message, upb_Arena* message_arena, - const upb_MiniTableExtension* ext, - upb_Message* extension, upb_Arena* extension_arena) { - if (message_arena != extension_arena && - // Try fuse, if fusing is not allowed or fails, create copy of extension. - !upb_Arena_Fuse(message_arena, extension_arena)) { - extension = DeepClone(extension, upb_MiniTableExtension_GetSubMessage(ext), - message_arena); - } - return upb_Message_SetExtension(message, ext, &extension, message_arena) - ? absl::OkStatus() - : MessageAllocationError(); -} - -absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, - const upb_MiniTableExtension* ext, - const upb_Message* extension) { - // Clone extension into target message arena. - extension = DeepClone(extension, upb_MiniTableExtension_GetSubMessage(ext), - message_arena); - return upb_Message_SetExtension(message, ext, &extension, message_arena) - ? absl::OkStatus() - : MessageAllocationError(); -} - -} // namespace internal - -} // namespace protos From 162a74067058a298ea1dc9ed7c0791b4c6abb69a Mon Sep 17 00:00:00 2001 From: Tony Liao Date: Tue, 23 Jul 2024 08:50:08 -0700 Subject: [PATCH 061/107] Reduced nesting in GenerateByteSize: slight readability improvements in generated code. GenerateByteSize itself remains deeply nested, but by factoring out one part of the loop, at least we make the part that generates UpdateByteSize a bit more readable. Making the callsite of MayEmitIfNonDefaultCheck less nested actually resulted in slight readability improvements also in the generated code, namely of the form: @@ -10563,8 +10559,7 @@ PROTOBUF_NOINLINE void OneStringEdition: { // string data = 1; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::proto2::internal::WireFormatLite::StringSize( this_._internal_data()); These readability improvements should be kept IMO -- they make the generated protobuf C++ code slightly easier to read. PiperOrigin-RevId: 655180880 --- .../golden/compare_cpp_codegen_failure.txt | 4 +- .../golden/compare_cpp_codegen_failure.xml | 2 +- src/google/protobuf/compiler/cpp/message.cc | 107 +++++++++--------- src/google/protobuf/compiler/cpp/message.h | 4 + src/google/protobuf/descriptor.pb.cc | 3 +- 5 files changed, 59 insertions(+), 61 deletions(-) diff --git a/editions/golden/compare_cpp_codegen_failure.txt b/editions/golden/compare_cpp_codegen_failure.txt index ff5fbdaffc..ab0bad33f1 100644 --- a/editions/golden/compare_cpp_codegen_failure.txt +++ b/editions/golden/compare_cpp_codegen_failure.txt @@ -30,9 +30,9 @@ { - // optional int32 int32_field = 1; + // int32 int32_field = 1; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( [ FAILED ] third_party/protobuf/editions/golden/simple_proto3.pb.cc [ RUN ] third_party/protobuf/editions/golden/simple_proto3.pb.h @@ @@ diff --git a/editions/golden/compare_cpp_codegen_failure.xml b/editions/golden/compare_cpp_codegen_failure.xml index a432b5a011..f992058c2c 100644 --- a/editions/golden/compare_cpp_codegen_failure.xml +++ b/editions/golden/compare_cpp_codegen_failure.xml @@ -2,7 +2,7 @@ - + diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 9a3a3ba80a..8384a5c70e 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -1213,6 +1213,55 @@ class AccessorVerifier { } // namespace +void MessageGenerator::EmitUpdateByteSizeForField( + const FieldDescriptor* field, io::Printer* p, + int& cached_has_word_index) const { + p->Emit( + {{"comment", [&] { PrintFieldComment(Formatter{p}, field, options_); }}, + {"update_byte_size_for_field", + [&] { field_generators_.get(field).GenerateByteSize(p); }}, + {"update_cached_has_bits", + [&] { + if (!HasHasbit(field) || field->options().weak()) return; + + int has_bit_index = has_bit_indices_[field->index()]; + + if (cached_has_word_index == (has_bit_index / 32)) return; + + cached_has_word_index = (has_bit_index / 32); + p->Emit({{"index", cached_has_word_index}}, + R"cc( + cached_has_bits = this_.$has_bits$[$index$]; + )cc"); + }}, + {"check_if_field_present", + [&] { + if (!HasHasbit(field)) { + MayEmitIfNonDefaultCheck(p, "this_.", field); + return; + } + + if (field->options().weak()) { + p->Emit("if (has_$name$())"); + return; + } + + int has_bit_index = has_bit_indices_[field->index()]; + p->Emit( + {{"mask", absl::StrFormat("0x%08xu", + uint32_t{1} << (has_bit_index % 32))}}, + "if (cached_has_bits & $mask$)"); + }}}, + R"cc( + $comment$; + $update_cached_has_bits$; + $check_if_field_present$ { + //~ Force newline. + $update_byte_size_for_field$; + } + )cc"); +} + void MessageGenerator::GenerateFieldAccessorDefinitions(io::Printer* p) { p->Emit("// $classname$\n\n"); @@ -4748,62 +4797,8 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) { // Go back and emit checks for each of the fields we // processed. for (const auto* field : fields) { - p->Emit( - {{"comment", - [&] { - PrintFieldComment(Formatter{p}, field, - options_); - }}, - {"update_byte_size_for_field", - [&] { - field_generators_.get(field).GenerateByteSize( - p); - }}, - {"update_cached_has_bits", - [&] { - if (!HasHasbit(field) || - field->options().weak()) - return; - int has_bit_index = - has_bit_indices_[field->index()]; - if (cached_has_word_index == - (has_bit_index / 32)) - return; - cached_has_word_index = (has_bit_index / 32); - p->Emit({{"index", cached_has_word_index}}, - R"cc( - cached_has_bits = - this_.$has_bits$[$index$]; - )cc"); - }}, - {"check_if_field_present", - [&] { - if (!HasHasbit(field)) { - MayEmitIfNonDefaultCheck(p, "this_.", field); - return; - } - - if (field->options().weak()) { - p->Emit("if (has_$name$())"); - return; - } - - int has_bit_index = - has_bit_indices_[field->index()]; - p->Emit( - {{"mask", absl::StrFormat( - "0x%08xu", - 1u << (has_bit_index % 32))}}, - "if (cached_has_bits & $mask$)"); - }}}, - R"cc( - $comment$; - $update_cached_has_bits$; - $check_if_field_present$ { - //~ Force newline. - $update_byte_size_for_field$; - } - )cc"); + EmitUpdateByteSizeForField(field, p, + cached_has_word_index); } }}, {"may_update_cached_has_word_index", diff --git a/src/google/protobuf/compiler/cpp/message.h b/src/google/protobuf/compiler/cpp/message.h index 4e5afb1084..11db273a08 100644 --- a/src/google/protobuf/compiler/cpp/message.h +++ b/src/google/protobuf/compiler/cpp/message.h @@ -186,6 +186,10 @@ class MessageGenerator { int HasWordIndex(const FieldDescriptor* field) const; std::vector RequiredFieldsBitMask() const; + // Helper function to reduce nesting levels of deep Emit calls. + void EmitUpdateByteSizeForField(const FieldDescriptor* field, io::Printer* p, + int& cached_has_word_index) const; + const Descriptor* descriptor_; int index_in_file_messages_; Options options_; diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index 4b3f6ce1b5..63caf08785 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -10530,8 +10530,7 @@ PROTOBUF_NOINLINE void OneofOptions::Clear() { } { // optional .google.protobuf.FeatureSet features = 1; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.features_); From 55e997ea222561c606ba7c59706e6377776c5d9b Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Tue, 23 Jul 2024 09:00:51 -0700 Subject: [PATCH 062/107] [ObjC] Deprecate GPBTextFormatForUnknownFieldSet(). `GPBTextFormatForMessage()` will contain any information for unknown fields. And it clears the way for `GPBUnknownFieldSet` to eventually be deprecated also. PiperOrigin-RevId: 655184316 --- objectivec/GPBUnknownFieldSet.m | 3 + objectivec/GPBUtilities.h | 4 +- objectivec/GPBUtilities.m | 84 ++++++++++++++++++++++++++-- objectivec/Tests/GPBUtilitiesTests.m | 57 +++++++++++++++++-- 4 files changed, 138 insertions(+), 10 deletions(-) diff --git a/objectivec/GPBUnknownFieldSet.m b/objectivec/GPBUnknownFieldSet.m index 5fe8ce03c4..10fb80c0c7 100644 --- a/objectivec/GPBUnknownFieldSet.m +++ b/objectivec/GPBUnknownFieldSet.m @@ -154,7 +154,10 @@ static void CopyWorker(__unused const void *key, const void *value, void *contex - (NSString *)description { NSMutableString *description = [NSMutableString stringWithFormat:@"<%@ %p>: TextFormat: {\n", [self class], self]; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" NSString *textFormat = GPBTextFormatForUnknownFieldSet(self, @" "); +#pragma clang diagnostic pop [description appendString:textFormat]; [description appendString:@"}"]; return description; diff --git a/objectivec/GPBUtilities.h b/objectivec/GPBUtilities.h index 9673aa4261..578545df05 100644 --- a/objectivec/GPBUtilities.h +++ b/objectivec/GPBUtilities.h @@ -40,7 +40,9 @@ NSString *GPBTextFormatForMessage(GPBMessage *message, NSString *__nullable line * @return An NSString with the TextFormat of the unknown field set. **/ NSString *GPBTextFormatForUnknownFieldSet(GPBUnknownFieldSet *__nullable unknownSet, - NSString *__nullable lineIndent); + NSString *__nullable lineIndent) + __attribute__((deprecated("Use GPBTextFormatForMessage will include the unknown fields, and " + "GPBUnknownFieldSet it going away."))); /** * Checks if the given field number is set on a message. diff --git a/objectivec/GPBUtilities.m b/objectivec/GPBUtilities.m index ff309860cd..0764c96910 100644 --- a/objectivec/GPBUtilities.m +++ b/objectivec/GPBUtilities.m @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#import "GPBUtilities_PackagePrivate.h" +#import "GPBUtilities.h" #import @@ -15,6 +15,9 @@ #import "GPBMessage_PackagePrivate.h" #import "GPBUnknownField.h" #import "GPBUnknownFieldSet.h" +#import "GPBUnknownField_PackagePrivate.h" +#import "GPBUnknownFields.h" +#import "GPBUtilities_PackagePrivate.h" // Direct access is use for speed, to avoid even internally declaring things // read/write, etc. The warning is enabled in the project to ensure code calling @@ -1929,6 +1932,78 @@ static void AppendTextFormatForMessageExtensionRange(GPBMessage *message, NSArra } // for..in(activeExtensions) } +static void AppendTextFormatForUnknownFields(GPBUnknownFields *ufs, NSMutableString *toStr, + NSString *lineIndent) { +#if defined(DEBUG) && DEBUG + NSCAssert(!ufs.empty, @"Internal Error: No unknown fields to format."); +#endif + // Extract the fields and sort them by field number. Use a stable sort and sort just by the field + // numbers, that way the output will still show the order the different types were added as well + // as maintaining the order within a give number/type pair (i.e. - repeated fields say in order). + NSMutableArray *sortedFields = [[NSMutableArray alloc] initWithCapacity:ufs.count]; + for (GPBUnknownField *field in ufs) { + [sortedFields addObject:field]; + } + [sortedFields + sortWithOptions:NSSortStable + usingComparator:^NSComparisonResult(GPBUnknownField *field1, GPBUnknownField *field2) { + int32_t fieldNumber1 = field1->number_; + int32_t fieldNumber2 = field2->number_; + if (fieldNumber1 < fieldNumber2) { + return NSOrderedAscending; + } else if (fieldNumber1 > fieldNumber2) { + return NSOrderedDescending; + } else { + return NSOrderedSame; + } + }]; + + NSString *subIndent = nil; + + for (GPBUnknownField *field in sortedFields) { + int32_t fieldNumber = field->number_; + switch (field->type_) { + case GPBUnknownFieldTypeVarint: + [toStr appendFormat:@"%@%d: %llu\n", lineIndent, fieldNumber, field->storage_.intValue]; + break; + case GPBUnknownFieldTypeFixed32: + [toStr appendFormat:@"%@%d: 0x%X\n", lineIndent, fieldNumber, + (uint32_t)field->storage_.intValue]; + break; + case GPBUnknownFieldTypeFixed64: + [toStr appendFormat:@"%@%d: 0x%llX\n", lineIndent, fieldNumber, field->storage_.intValue]; + break; + case GPBUnknownFieldTypeLengthDelimited: + [toStr appendFormat:@"%@%d: ", lineIndent, fieldNumber]; + AppendBufferAsString(field->storage_.lengthDelimited, toStr); + [toStr appendString:@"\n"]; + break; + case GPBUnknownFieldTypeGroup: { + GPBUnknownFields *group = field->storage_.group; + if (group.empty) { + [toStr appendFormat:@"%@%d: {}\n", lineIndent, fieldNumber]; + } else { + [toStr appendFormat:@"%@%d: {\n", lineIndent, fieldNumber]; + if (subIndent == nil) { + subIndent = [lineIndent stringByAppendingString:@" "]; + } + AppendTextFormatForUnknownFields(group, toStr, subIndent); + [toStr appendFormat:@"%@}\n", lineIndent]; + } + } break; + case GPBUnknownFieldTypeLegacy: +#if defined(DEBUG) && DEBUG + NSCAssert( + NO, + @"Internal error: Shouldn't have gotten a legacy field type in the unknown fields."); +#endif + break; + } + } + [subIndent release]; + [sortedFields release]; +} + static void AppendTextFormatForMessage(GPBMessage *message, NSMutableString *toStr, NSString *lineIndent) { GPBDescriptor *descriptor = [message descriptor]; @@ -1951,11 +2026,12 @@ static void AppendTextFormatForMessage(GPBMessage *message, NSMutableString *toS } } - NSString *unknownFieldsStr = GPBTextFormatForUnknownFieldSet(message.unknownFields, lineIndent); - if ([unknownFieldsStr length] > 0) { + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:message]; + if (ufs.count > 0) { [toStr appendFormat:@"%@# --- Unknown fields ---\n", lineIndent]; - [toStr appendString:unknownFieldsStr]; + AppendTextFormatForUnknownFields(ufs, toStr, lineIndent); } + [ufs release]; } NSString *GPBTextFormatForMessage(GPBMessage *message, NSString *lineIndent) { diff --git a/objectivec/Tests/GPBUtilitiesTests.m b/objectivec/Tests/GPBUtilitiesTests.m index e32fccda46..d05a345523 100644 --- a/objectivec/Tests/GPBUtilitiesTests.m +++ b/objectivec/Tests/GPBUtilitiesTests.m @@ -6,17 +6,16 @@ // https://developers.google.com/open-source/licenses/bsd #import - -#import "GPBUtilities_PackagePrivate.h" - #import -#import "GPBTestUtilities.h" - #import "GPBDescriptor.h" #import "GPBDescriptor_PackagePrivate.h" #import "GPBMessage.h" +#import "GPBTestUtilities.h" +#import "GPBUnknownField.h" #import "GPBUnknownField_PackagePrivate.h" +#import "GPBUtilities.h" +#import "GPBUtilities_PackagePrivate.h" #import "objectivec/Tests/MapUnittest.pbobjc.h" #import "objectivec/Tests/Unittest.pbobjc.h" @@ -168,6 +167,54 @@ [expected release]; } +- (void)testTextFormatUnknownFields { + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:100 varint:5]; + [ufs addFieldNumber:100 varint:4]; + [ufs addFieldNumber:10 varint:1]; + [ufs addFieldNumber:300 fixed32:0x50]; + [ufs addFieldNumber:300 fixed32:0x40]; + [ufs addFieldNumber:10 fixed32:0x10]; + [ufs addFieldNumber:200 fixed64:0x5000]; + [ufs addFieldNumber:200 fixed64:0x4000]; + [ufs addFieldNumber:10 fixed64:0x1000]; + [ufs addFieldNumber:10 lengthDelimited:DataFromCStr("foo")]; + [ufs addFieldNumber:10 lengthDelimited:DataFromCStr("bar")]; + GPBUnknownFields *group = [ufs addGroupWithFieldNumber:150]; + [group addFieldNumber:2 varint:2]; + [group addFieldNumber:1 varint:1]; + group = [ufs addGroupWithFieldNumber:150]; + [group addFieldNumber:1 varint:1]; + [group addFieldNumber:3 fixed32:0x3]; + [group addFieldNumber:2 fixed64:0x2]; + TestEmptyMessage *message = [TestEmptyMessage message]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; + + NSString *expected = @"# --- Unknown fields ---\n" + @"10: 1\n" + @"10: 0x10\n" + @"10: 0x1000\n" + @"10: \"foo\"\n" + @"10: \"bar\"\n" + @"100: 5\n" + @"100: 4\n" + @"150: {\n" + @" 1: 1\n" + @" 2: 2\n" + @"}\n" + @"150: {\n" + @" 1: 1\n" + @" 2: 0x2\n" + @" 3: 0x3\n" + @"}\n" + @"200: 0x5000\n" + @"200: 0x4000\n" + @"300: 0x50\n" + @"300: 0x40\n"; + NSString *result = GPBTextFormatForMessage(message, nil); + XCTAssertEqualObjects(expected, result); +} + - (void)testSetRepeatedFields { TestAllTypes *message = [TestAllTypes message]; From af3012b66d7a109eab77b1db6268209e2df83ec1 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Jul 2024 16:04:34 +0000 Subject: [PATCH 063/107] Auto-generate files after cl/655180880 --- src/google/protobuf/api.pb.cc | 3 +-- src/google/protobuf/type.pb.cc | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index 3715740939..e9d38dcdf6 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -602,8 +602,7 @@ PROTOBUF_NOINLINE void Api::Clear() { } { // .google.protobuf.SourceContext source_context = 5; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.source_context_); diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index fbaae9d642..dbf5dc060c 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -793,8 +793,7 @@ PROTOBUF_NOINLINE void Type::Clear() { } { // .google.protobuf.SourceContext source_context = 5; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.source_context_); @@ -1679,8 +1678,7 @@ PROTOBUF_NOINLINE void Enum::Clear() { } { // .google.protobuf.SourceContext source_context = 4; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.source_context_); @@ -2269,8 +2267,7 @@ PROTOBUF_NOINLINE void Option::Clear() { } { // .google.protobuf.Any value = 2; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.value_); From 5ceae5f3b0f4792fe8f6f0856308fffcc2cbc24f Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Tue, 23 Jul 2024 09:36:08 -0700 Subject: [PATCH 064/107] [ObjC] Update tests to use both unknown fields apis. Also fixes edge case where merging into an autocreated message for a field wasn't marking the field as set in the parent. PiperOrigin-RevId: 655196339 --- objectivec/GPBMessage.m | 1 + objectivec/Tests/GPBCodedInputStreamTests.m | 41 +++++++++++++- .../Tests/GPBMessageTests+Serialization.m | 17 ++++++ objectivec/Tests/GPBMessageTests.m | 55 +++++++++++++++++++ objectivec/Tests/GPBUnknownFieldSetTest.m | 4 ++ objectivec/Tests/GPBUnknownFieldsTest.m | 1 - objectivec/Tests/GPBUtilitiesTests.m | 36 ++++++++++++ 7 files changed, 152 insertions(+), 3 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 9ef625b01f..1761eab485 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -2085,6 +2085,7 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { - (BOOL)mergeFromData:(NSData *)data extensionRegistry:(nullable id)extensionRegistry error:(NSError **)errorPtr { + GPBBecomeVisibleToAutocreator(self); GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; @try { [self mergeFromCodedInputStream:input extensionRegistry:extensionRegistry endingTag:0]; diff --git a/objectivec/Tests/GPBCodedInputStreamTests.m b/objectivec/Tests/GPBCodedInputStreamTests.m index 6bf775dded..29802eb15a 100644 --- a/objectivec/Tests/GPBCodedInputStreamTests.m +++ b/objectivec/Tests/GPBCodedInputStreamTests.m @@ -11,7 +11,9 @@ #import "GPBCodedInputStream_PackagePrivate.h" #import "GPBCodedOutputStream.h" #import "GPBTestUtilities.h" +#import "GPBUnknownField.h" #import "GPBUnknownFieldSet_PackagePrivate.h" +#import "GPBUnknownFields.h" #import "GPBUtilities_PackagePrivate.h" #import "GPBWireFormat.h" #import "objectivec/Tests/Unittest.pbobjc.h" @@ -268,20 +270,55 @@ TestAllTypes* message = [self allSetRepeatedCount:kGPBDefaultRepeatCount]; NSData* rawBytes = message.data; - // Create two parallel inputs. Parse one as unknown fields while using - // skipField() to skip each field on the other. Expect the same tags. + TestEmptyMessage* empty = [TestEmptyMessage parseFromData:rawBytes error:NULL]; + XCTAssertNotNil(empty); + GPBUnknownFields* ufs = [[[GPBUnknownFields alloc] initFromMessage:empty] autorelease]; + NSMutableArray* fieldNumbers = [NSMutableArray arrayWithCapacity:ufs.count]; + for (GPBUnknownField* field in ufs) { + GPBWireFormat wireFormat; + switch (field.type) { + case GPBUnknownFieldTypeFixed32: + wireFormat = GPBWireFormatFixed32; + break; + case GPBUnknownFieldTypeFixed64: + wireFormat = GPBWireFormatFixed64; + break; + case GPBUnknownFieldTypeVarint: + wireFormat = GPBWireFormatVarint; + break; + case GPBUnknownFieldTypeLengthDelimited: + wireFormat = GPBWireFormatLengthDelimited; + break; + case GPBUnknownFieldTypeGroup: + wireFormat = GPBWireFormatStartGroup; + break; + case GPBUnknownFieldTypeLegacy: + XCTFail(@"Legacy field type not expected"); + wireFormat = GPBWireFormatVarint; + break; + } + uint32_t tag = GPBWireFormatMakeTag(field.number, wireFormat); + [fieldNumbers addObject:@(tag)]; + } + + // Check the tags compared to what's in the UnknownFields to confirm the stream is + // skipping as expected (this covers the tags within a group also). GPBCodedInputStream* input1 = [GPBCodedInputStream streamWithData:rawBytes]; GPBCodedInputStream* input2 = [GPBCodedInputStream streamWithData:rawBytes]; GPBUnknownFieldSet* unknownFields = [[[GPBUnknownFieldSet alloc] init] autorelease]; + NSUInteger idx = 0; while (YES) { int32_t tag = [input1 readTag]; XCTAssertEqual(tag, [input2 readTag]); if (tag == 0) { + XCTAssertEqual(idx, fieldNumbers.count); break; } + XCTAssertEqual(tag, [fieldNumbers[idx] intValue]); [unknownFields mergeFieldFrom:tag input:input1]; [input2 skipField:tag]; + ++idx; } } diff --git a/objectivec/Tests/GPBMessageTests+Serialization.m b/objectivec/Tests/GPBMessageTests+Serialization.m index f3e8abce15..b42bfdd564 100644 --- a/objectivec/Tests/GPBMessageTests+Serialization.m +++ b/objectivec/Tests/GPBMessageTests+Serialization.m @@ -419,6 +419,17 @@ // All the values should be in unknown fields. + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:msg]; + XCTAssertEqual(ufs.count, 3U); + uint64_t varint; + XCTAssertTrue([ufs getFirst:Message2_FieldNumber_OptionalEnum varint:&varint]); + XCTAssertEqual(varint, (uint64_t)Message3_Enum_Extra3); + XCTAssertTrue([ufs getFirst:Message2_FieldNumber_RepeatedEnumArray varint:&varint]); + XCTAssertEqual(varint, (uint64_t)Message3_Enum_Extra3); + XCTAssertTrue([ufs getFirst:Message2_FieldNumber_OneofEnum varint:&varint]); + XCTAssertEqual(varint, (uint64_t)Message3_Enum_Extra3); + [ufs release]; + GPBUnknownFieldSet *unknownFields = msg.unknownFields; XCTAssertEqual([unknownFields countOfFields], 3U); @@ -1398,6 +1409,9 @@ int32_t val = -1; XCTAssertTrue([msg1.knownMapField getEnum:&val forKey:0]); XCTAssertEqual(val, Proto2MapEnum_Proto2MapEnumFoo); + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:msg1]; + XCTAssertEqual(ufs.count, 1U); + [ufs release]; XCTAssertEqual(msg1.unknownFields.countOfFields, 1U); data = [msg1 data]; @@ -1410,6 +1424,9 @@ XCTAssertEqual(msg2.unknownMapField.count, 1U); XCTAssertTrue([msg2.unknownMapField getEnum:&val forKey:0]); XCTAssertEqual(val, Proto2MapEnumPlusExtra_EProto2MapEnumExtra); + ufs = [[GPBUnknownFields alloc] initFromMessage:msg2]; + XCTAssertTrue(ufs.empty); + [ufs release]; XCTAssertEqual(msg2.unknownFields.countOfFields, 0U); XCTAssertEqualObjects(orig, msg2); diff --git a/objectivec/Tests/GPBMessageTests.m b/objectivec/Tests/GPBMessageTests.m index e70f898e3e..f3272dbaf2 100644 --- a/objectivec/Tests/GPBMessageTests.m +++ b/objectivec/Tests/GPBMessageTests.m @@ -13,8 +13,10 @@ #import "GPBDictionary_PackagePrivate.h" #import "GPBMessage_PackagePrivate.h" #import "GPBTestUtilities.h" +#import "GPBUnknownField.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" +#import "GPBUnknownFields.h" #import "objectivec/Tests/Unittest.pbobjc.h" #import "objectivec/Tests/UnittestImport.pbobjc.h" #import "objectivec/Tests/UnittestObjc.pbobjc.h" @@ -501,6 +503,11 @@ [message setUnknownFields:unknownFields]; + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:1234 fixed32:1234]; + [ufs addFieldNumber:2345 varint:54321]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; + NSString *description = [message description]; XCTAssertGreaterThan([description length], 0U); @@ -985,6 +992,19 @@ XCTAssertFalse([message hasOptionalNestedMessage]); [message.optionalNestedMessage setUnknownFields:unknownFields]; XCTAssertTrue([message hasOptionalNestedMessage]); + + message.optionalNestedMessage = nil; + XCTAssertFalse([message hasOptionalNestedMessage]); + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:1 varint:1]; + [message.optionalNestedMessage mergeUnknownFields:ufs extensionRegistry:nil]; + XCTAssertTrue([message hasOptionalNestedMessage]); + + message.optionalNestedMessage = nil; + XCTAssertFalse([message hasOptionalNestedMessage]); + [ufs clear]; // Also make sure merging zero length forces it to become visible. + [message.optionalNestedMessage mergeUnknownFields:ufs extensionRegistry:nil]; + XCTAssertTrue([message hasOptionalNestedMessage]); } - (void)testSetAutocreatedSubmessageToSelf { @@ -1481,6 +1501,19 @@ XCTAssertFalse([msg hasExtension:[UnittestRoot repeatedNestedEnumExtension]]); XCTAssertFalse([msg hasExtension:[UnittestRoot repeatedForeignEnumExtension]]); + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] initFromMessage:msg] autorelease]; + XCTAssertEqual(ufs.count, 3); + uint64_t varint; + XCTAssertTrue([ufs getFirst:[UnittestRoot optionalNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 10); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 11); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedForeignEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 12); + GPBUnknownFieldSet *unknownFields = msg.unknownFields; GPBUnknownField *field = [unknownFields getField:[UnittestRoot optionalNestedEnumExtension].fieldNumber]; @@ -1523,6 +1556,18 @@ expected = @[ @4, @6 ]; XCTAssertEqualObjects([msg getExtension:[UnittestRoot repeatedForeignEnumExtension]], expected); + ufs = [[[GPBUnknownFields alloc] initFromMessage:msg] autorelease]; + XCTAssertEqual(ufs.count, 3); + XCTAssertTrue([ufs getFirst:[UnittestRoot optionalNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 10); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 11); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedForeignEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 12); + unknownFields = msg.unknownFields; field = [unknownFields getField:[UnittestRoot optionalNestedEnumExtension].fieldNumber]; XCTAssertNotNil(field); @@ -1840,6 +1885,9 @@ [unknowns mergeVarintField:123 value:456]; GPBMessage *message = [GPBMessage message]; [message setUnknownFields:unknowns]; + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:1234 varint:5678]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; NSData *data = [message data]; GPBMessage *message2 = [GPBMessage parseFromData:data extensionRegistry:nil error:NULL]; XCTAssertEqualObjects(message, message2); @@ -1850,12 +1898,19 @@ [unknowns1 mergeVarintField:123 value:456]; GPBMessage *message1 = [GPBMessage message]; [message1 setUnknownFields:unknowns1]; + GPBUnknownFields *ufs1 = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs1 addFieldNumber:1234 varint:5678]; + [message1 mergeUnknownFields:ufs1 extensionRegistry:nil]; GPBUnknownFieldSet *unknowns2 = [[[GPBUnknownFieldSet alloc] init] autorelease]; [unknowns2 mergeVarintField:789 value:987]; [unknowns2 mergeVarintField:654 value:321]; GPBMessage *message2 = [GPBMessage message]; [message2 setUnknownFields:unknowns2]; + GPBUnknownFields *ufs2 = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs2 addFieldNumber:2345 fixed32:6789]; + [ufs2 addFieldNumber:3456 fixed32:7890]; + [message2 mergeUnknownFields:ufs2 extensionRegistry:nil]; NSMutableData *delimitedData = [NSMutableData data]; [delimitedData appendData:[message1 delimitedData]]; diff --git a/objectivec/Tests/GPBUnknownFieldSetTest.m b/objectivec/Tests/GPBUnknownFieldSetTest.m index 1e055b0dd0..890ce8f4ad 100644 --- a/objectivec/Tests/GPBUnknownFieldSetTest.m +++ b/objectivec/Tests/GPBUnknownFieldSetTest.m @@ -282,6 +282,10 @@ TestEmptyMessage* destination2 = [TestEmptyMessage message]; [destination2 mergeFrom:source3]; + XCTAssertEqualObjects(destination1.unknownFields, destination2.unknownFields); + XCTAssertEqualObjects(destination1.unknownFields, source3.unknownFields); + XCTAssertEqualObjects(destination2.unknownFields, source3.unknownFields); + XCTAssertEqualObjects(destination1.data, destination2.data); XCTAssertEqualObjects(destination1.data, source3.data); XCTAssertEqualObjects(destination2.data, source3.data); diff --git a/objectivec/Tests/GPBUnknownFieldsTest.m b/objectivec/Tests/GPBUnknownFieldsTest.m index 5565f674b5..8c2adcf32c 100644 --- a/objectivec/Tests/GPBUnknownFieldsTest.m +++ b/objectivec/Tests/GPBUnknownFieldsTest.m @@ -918,7 +918,6 @@ static NSData* DataForGroupsOfDepth(NSUInteger depth) { uint64_t fixed64 = 0; XCTAssertTrue([group getFirst:3 fixed64:&fixed64]); XCTAssertEqual(fixed64, 0x123456789abcdef0LL); - XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36) error:NULL]; // length delimited, length 0 diff --git a/objectivec/Tests/GPBUtilitiesTests.m b/objectivec/Tests/GPBUtilitiesTests.m index d05a345523..fb0175ecca 100644 --- a/objectivec/Tests/GPBUtilitiesTests.m +++ b/objectivec/Tests/GPBUtilitiesTests.m @@ -251,11 +251,27 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { return result; } +// Helper to add an unknown field data to messages. +static void AddUnknownFields(GPBMessage *message, int num) { + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] init]; + [ufs addFieldNumber:num varint:num]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; + [ufs release]; +} + +static BOOL HasUnknownFields(GPBMessage *message) { + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:message]; + BOOL result = !ufs.empty; + [ufs release]; + return result; +} + - (void)testDropMessageUnknownFieldsRecursively { TestAllExtensions *message = [TestAllExtensions message]; // Give it unknownFields. message.unknownFields = UnknownFieldsSetHelper(777); + AddUnknownFields(message, 1777); // Given it extensions that include a message with unknown fields of its own. { @@ -266,18 +282,21 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { OptionalGroup_extension *optionalGroup = [OptionalGroup_extension message]; optionalGroup.a = 123; optionalGroup.unknownFields = UnknownFieldsSetHelper(779); + AddUnknownFields(optionalGroup, 1779); [message setExtension:[UnittestRoot optionalGroupExtension] value:optionalGroup]; // Message TestAllTypes_NestedMessage *nestedMessage = [TestAllTypes_NestedMessage message]; nestedMessage.bb = 456; nestedMessage.unknownFields = UnknownFieldsSetHelper(778); + AddUnknownFields(nestedMessage, 1778); [message setExtension:[UnittestRoot optionalNestedMessageExtension] value:nestedMessage]; // Repeated Group RepeatedGroup_extension *repeatedGroup = [[RepeatedGroup_extension alloc] init]; repeatedGroup.a = 567; repeatedGroup.unknownFields = UnknownFieldsSetHelper(780); + AddUnknownFields(repeatedGroup, 1780); [message addExtension:[UnittestRoot repeatedGroupExtension] value:repeatedGroup]; [repeatedGroup release]; @@ -285,6 +304,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { nestedMessage = [[TestAllTypes_NestedMessage alloc] init]; nestedMessage.bb = 678; nestedMessage.unknownFields = UnknownFieldsSetHelper(781); + AddUnknownFields(nestedMessage, 1781); [message addExtension:[UnittestRoot repeatedNestedMessageExtension] value:nestedMessage]; [nestedMessage release]; } @@ -292,6 +312,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { // Confirm everything is there. XCTAssertNotNil(message); + XCTAssertTrue(HasUnknownFields(message)); XCTAssertNotNil(message.unknownFields); XCTAssertTrue([message hasExtension:[UnittestRoot optionalInt32Extension]]); @@ -301,6 +322,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalGroupExtension]]; XCTAssertNotNil(optionalGroup); XCTAssertEqual(optionalGroup.a, 123); + XCTAssertTrue(HasUnknownFields(optionalGroup)); XCTAssertNotNil(optionalGroup.unknownFields); } @@ -310,6 +332,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalNestedMessageExtension]]; XCTAssertNotNil(nestedMessage); XCTAssertEqual(nestedMessage.bb, 456); + XCTAssertTrue(HasUnknownFields(nestedMessage)); XCTAssertNotNil(nestedMessage.unknownFields); } @@ -320,6 +343,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { RepeatedGroup_extension *repeatedGroup = repeatedGroups.firstObject; XCTAssertNotNil(repeatedGroup); XCTAssertEqual(repeatedGroup.a, 567); + XCTAssertTrue(HasUnknownFields(repeatedGroup)); XCTAssertNotNil(repeatedGroup.unknownFields); } @@ -331,6 +355,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { TestAllTypes_NestedMessage *repeatedNestedMessage = repeatedNestedMessages.firstObject; XCTAssertNotNil(repeatedNestedMessage); XCTAssertEqual(repeatedNestedMessage.bb, 678); + XCTAssertTrue(HasUnknownFields(repeatedNestedMessage)); XCTAssertNotNil(repeatedNestedMessage.unknownFields); } @@ -340,6 +365,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { // Confirm unknowns are gone from within the messages. XCTAssertNotNil(message); + XCTAssertFalse(HasUnknownFields(message)); XCTAssertNil(message.unknownFields); XCTAssertTrue([message hasExtension:[UnittestRoot optionalInt32Extension]]); @@ -349,6 +375,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalGroupExtension]]; XCTAssertNotNil(optionalGroup); XCTAssertEqual(optionalGroup.a, 123); + XCTAssertFalse(HasUnknownFields(optionalGroup)); XCTAssertNil(optionalGroup.unknownFields); } @@ -358,6 +385,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalNestedMessageExtension]]; XCTAssertNotNil(nestedMessage); XCTAssertEqual(nestedMessage.bb, 456); + XCTAssertFalse(HasUnknownFields(nestedMessage)); XCTAssertNil(nestedMessage.unknownFields); } @@ -368,6 +396,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { RepeatedGroup_extension *repeatedGroup = repeatedGroups.firstObject; XCTAssertNotNil(repeatedGroup); XCTAssertEqual(repeatedGroup.a, 567); + XCTAssertFalse(HasUnknownFields(repeatedGroup)); XCTAssertNil(repeatedGroup.unknownFields); } @@ -379,6 +408,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { TestAllTypes_NestedMessage *repeatedNestedMessage = repeatedNestedMessages.firstObject; XCTAssertNotNil(repeatedNestedMessage); XCTAssertEqual(repeatedNestedMessage.bb, 678); + XCTAssertFalse(HasUnknownFields(repeatedNestedMessage)); XCTAssertNil(repeatedNestedMessage.unknownFields); } } @@ -389,10 +419,12 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { { ForeignMessage *foreignMessage = [ForeignMessage message]; foreignMessage.unknownFields = UnknownFieldsSetHelper(100); + AddUnknownFields(foreignMessage, 1000); [message.mapInt32ForeignMessage setObject:foreignMessage forKey:100]; foreignMessage = [ForeignMessage message]; foreignMessage.unknownFields = UnknownFieldsSetHelper(101); + AddUnknownFields(foreignMessage, 1001); [message.mapStringForeignMessage setObject:foreignMessage forKey:@"101"]; } @@ -403,12 +435,14 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { { ForeignMessage *foreignMessage = [message.mapInt32ForeignMessage objectForKey:100]; XCTAssertNotNil(foreignMessage); + XCTAssertTrue(HasUnknownFields(foreignMessage)); XCTAssertNotNil(foreignMessage.unknownFields); } { ForeignMessage *foreignMessage = [message.mapStringForeignMessage objectForKey:@"101"]; XCTAssertNotNil(foreignMessage); + XCTAssertTrue(HasUnknownFields(foreignMessage)); XCTAssertNotNil(foreignMessage.unknownFields); } @@ -421,12 +455,14 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { { ForeignMessage *foreignMessage = [message.mapInt32ForeignMessage objectForKey:100]; XCTAssertNotNil(foreignMessage); + XCTAssertFalse(HasUnknownFields(foreignMessage)); XCTAssertNil(foreignMessage.unknownFields); } { ForeignMessage *foreignMessage = [message.mapStringForeignMessage objectForKey:@"101"]; XCTAssertNotNil(foreignMessage); + XCTAssertFalse(HasUnknownFields(foreignMessage)); XCTAssertNil(foreignMessage.unknownFields); } } From 224573d66a0cc958c76cb43d8b2eb3aa7cdb89f2 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Tue, 23 Jul 2024 09:55:21 -0700 Subject: [PATCH 065/107] [ObjC] Internal helper for getting the unknown field data from a message PiperOrigin-RevId: 655202619 --- objectivec/GPBMessage.m | 19 +++++++++++++++++++ objectivec/GPBMessage_PackagePrivate.h | 3 +++ objectivec/GPBUnknownFields.m | 19 +++---------------- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 1761eab485..b0367699dc 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -3684,4 +3684,23 @@ id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { return expected; } +NSData *GPBMessageUnknownFieldsData(GPBMessage *self) { + NSData *result = nil; + GPBUnknownFieldSet *unknownFields = self->unknownFields_; + if (unknownFields) { + if (self.descriptor.isWireFormat) { + NSMutableData *mutableData = + [NSMutableData dataWithLength:unknownFields.serializedSizeAsMessageSet]; + GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; + [unknownFields writeAsMessageSetTo:output]; + [output flush]; + [output release]; + result = mutableData; + } else { + result = [unknownFields data]; + } + } + return result; +} + #pragma clang diagnostic pop diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index 65a1a7c5ba..58649c44f7 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -81,4 +81,7 @@ void GPBAutocreatedDictionaryModified(GPBMessage *self, id dictionary); // autocreated reference to this message. void GPBClearMessageAutocreator(GPBMessage *self); +// The data (or null) from the unknown fields of a message; +NSData *GPBMessageUnknownFieldsData(GPBMessage *self); + CF_EXTERN_C_END diff --git a/objectivec/GPBUnknownFields.m b/objectivec/GPBUnknownFields.m index f3073ad16d..836e249f67 100644 --- a/objectivec/GPBUnknownFields.m +++ b/objectivec/GPBUnknownFields.m @@ -14,6 +14,7 @@ #import "GPBCodedOutputStream_PackagePrivate.h" #import "GPBDescriptor.h" #import "GPBMessage.h" +#import "GPBMessage_PackagePrivate.h" #import "GPBUnknownField.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" @@ -196,22 +197,8 @@ static BOOL MergeFromInputStream(GPBUnknownFields *self, GPBCodedInputStream *in self = [super init]; if (self) { fields_ = [[NSMutableArray alloc] init]; - // TODO: b/349146447 - Move off the legacy class and directly to the data once Message is - // updated. - GPBUnknownFieldSet *legacyUnknownFields = [message unknownFields]; - if (legacyUnknownFields) { - NSData *data; - if (message.descriptor.isWireFormat) { - NSMutableData *mutableData = - [NSMutableData dataWithLength:legacyUnknownFields.serializedSizeAsMessageSet]; - GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; - [legacyUnknownFields writeAsMessageSetTo:output]; - [output flush]; - [output release]; - data = mutableData; - } else { - data = [legacyUnknownFields data]; - } + NSData *data = GPBMessageUnknownFieldsData(message); + if (data) { GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; // Parse until the end of the data (tag will be zero). if (!MergeFromInputStream(self, input, 0)) { From 607b4b1b6ce319359272e30a8b8f239fcb522588 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Jul 2024 10:28:15 -0700 Subject: [PATCH 066/107] Change rust/upb to only ever do `use super::` rather than `use crate::` This makes it more 'relocatable' as a module into a monolithic runtime crate. PiperOrigin-RevId: 655215778 --- rust/upb/arena.rs | 2 +- rust/upb/array.rs | 7 ++++--- rust/upb/extension_registry.rs | 2 +- rust/upb/map.rs | 7 ++++--- rust/upb/message.rs | 4 ++-- rust/upb/message_value.rs | 2 +- rust/upb/mini_table.rs | 2 +- rust/upb/owned_arena_box.rs | 2 +- rust/upb/text.rs | 2 +- rust/upb/wire.rs | 2 +- 10 files changed, 17 insertions(+), 15 deletions(-) diff --git a/rust/upb/arena.rs b/rust/upb/arena.rs index b1a90dc4e9..d4581eabd3 100644 --- a/rust/upb/arena.rs +++ b/rust/upb/arena.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; +use super::opaque_pointee::opaque_pointee; use std::alloc::{self, Layout}; use std::cell::UnsafeCell; use std::marker::PhantomData; diff --git a/rust/upb/array.rs b/rust/upb/array.rs index aeadd1bdfc..1c42cefbb0 100644 --- a/rust/upb/array.rs +++ b/rust/upb/array.rs @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; -use crate::{upb_MessageValue, upb_MutableMessageValue, CType, RawArena}; +use super::opaque_pointee::opaque_pointee; +use super::{upb_MessageValue, upb_MutableMessageValue, CType, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Array); @@ -27,13 +27,14 @@ extern "C" { #[cfg(test)] mod tests { + use super::super::Arena; use super::*; #[test] fn array_ffi_test() { // SAFETY: FFI unit test uses C API under expected patterns. unsafe { - let arena = crate::Arena::new(); + let arena = Arena::new(); let raw_arena = arena.raw(); let array = upb_Array_New(raw_arena, CType::Float); diff --git a/rust/upb/extension_registry.rs b/rust/upb/extension_registry.rs index c0400cd61e..ce7fd26b6e 100644 --- a/rust/upb/extension_registry.rs +++ b/rust/upb/extension_registry.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; +use super::opaque_pointee::opaque_pointee; use std::ptr::NonNull; opaque_pointee!(upb_ExtensionRegistry); diff --git a/rust/upb/map.rs b/rust/upb/map.rs index 5c97a21d64..5b60149a74 100644 --- a/rust/upb/map.rs +++ b/rust/upb/map.rs @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; -use crate::{upb_MessageValue, CType, RawArena}; +use super::opaque_pointee::opaque_pointee; +use super::{upb_MessageValue, CType, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Map); @@ -49,13 +49,14 @@ extern "C" { #[cfg(test)] mod tests { + use super::super::Arena; use super::*; #[test] fn map_ffi_test() { // SAFETY: FFI unit test uses C API under expected patterns. unsafe { - let arena = crate::Arena::new(); + let arena = Arena::new(); let raw_arena = arena.raw(); let map = upb_Map_New(raw_arena, CType::Bool, CType::Double); assert_eq!(upb_Map_Size(map), 0); diff --git a/rust/upb/message.rs b/rust/upb/message.rs index 865c52ee9d..67d1e06e78 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; -use crate::{upb_ExtensionRegistry, upb_MiniTable, upb_MiniTableField, RawArena}; +use super::opaque_pointee::opaque_pointee; +use super::{upb_ExtensionRegistry, upb_MiniTable, upb_MiniTableField, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Message); diff --git a/rust/upb/message_value.rs b/rust/upb/message_value.rs index 1bb5a8deb7..5cde4d6bb2 100644 --- a/rust/upb/message_value.rs +++ b/rust/upb/message_value.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::{RawArray, RawMap, RawMessage, StringView}; +use super::{RawArray, RawMap, RawMessage, StringView}; // Transcribed from google3/third_party/upb/upb/message/value.h #[repr(C)] diff --git a/rust/upb/mini_table.rs b/rust/upb/mini_table.rs index fe80a86a68..64b64ec192 100644 --- a/rust/upb/mini_table.rs +++ b/rust/upb/mini_table.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; +use super::opaque_pointee::opaque_pointee; use std::ptr::NonNull; opaque_pointee!(upb_MiniTable); diff --git a/rust/upb/owned_arena_box.rs b/rust/upb/owned_arena_box.rs index 4cd11d6666..c4ff1ac809 100644 --- a/rust/upb/owned_arena_box.rs +++ b/rust/upb/owned_arena_box.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::Arena; +use super::Arena; use std::fmt::{self, Debug}; use std::ops::{Deref, DerefMut}; use std::ptr::NonNull; diff --git a/rust/upb/text.rs b/rust/upb/text.rs index 2e6f84adaf..2f1cf09448 100644 --- a/rust/upb/text.rs +++ b/rust/upb/text.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::{upb_MiniTable, RawMessage}; +use super::{upb_MiniTable, RawMessage}; extern "C" { /// Returns the minimum needed length (excluding NULL) that `buf` has to be diff --git a/rust/upb/wire.rs b/rust/upb/wire.rs index eaf53653a4..fe45558118 100644 --- a/rust/upb/wire.rs +++ b/rust/upb/wire.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::{upb_ExtensionRegistry, upb_MiniTable, Arena, RawArena, RawMessage}; +use super::{upb_ExtensionRegistry, upb_MiniTable, Arena, RawArena, RawMessage}; // LINT.IfChange(encode_status) #[repr(C)] From 41f12a239c749cc88cf0ad9df3c786f4292788a0 Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Tue, 23 Jul 2024 10:56:09 -0700 Subject: [PATCH 067/107] Add missing #![deny(unsafe_op_in_unsafe_fn)] to upb/rust/lib.rs and fix violating functions PiperOrigin-RevId: 655226304 --- rust/upb/lib.rs | 2 ++ rust/upb/wire.rs | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index bbfc106b6b..650baec854 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -4,8 +4,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#![deny(unsafe_op_in_unsafe_fn)] mod arena; + pub use arena::{upb_Arena, Arena, RawArena}; mod array; diff --git a/rust/upb/wire.rs b/rust/upb/wire.rs index fe45558118..75e666f9d1 100644 --- a/rust/upb/wire.rs +++ b/rust/upb/wire.rs @@ -56,12 +56,12 @@ pub unsafe fn encode( // SAFETY: // - `mini_table` is the one associated with `msg`. // - `buf` and `buf_size` are legally writable. - let status = upb_Encode(msg, mini_table, 0, arena.raw(), &mut buf, &mut len); + let status = unsafe { upb_Encode(msg, mini_table, 0, arena.raw(), &mut buf, &mut len) }; if status == EncodeStatus::Ok { assert!(!buf.is_null()); // EncodeStatus Ok should never return NULL data, even for len=0. // SAFETY: upb guarantees that `buf` is valid to read for `len`. - Ok((*std::ptr::slice_from_raw_parts(buf, len)).to_vec()) + Ok(unsafe { &*std::ptr::slice_from_raw_parts(buf, len) }.to_vec()) } else { Err(status) } @@ -87,7 +87,8 @@ pub unsafe fn decode( // - `mini_table` is the one associated with `msg` // - `buf` is legally readable for at least `buf_size` bytes. // - `extreg` is null. - let status = upb_Decode(buf, len, msg, mini_table, std::ptr::null(), options, arena.raw()); + let status = + unsafe { upb_Decode(buf, len, msg, mini_table, std::ptr::null(), options, arena.raw()) }; match status { DecodeStatus::Ok => Ok(()), _ => Err(status), From 9e34d5f55906b706e6a744211de5bb3377e81694 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Jul 2024 13:50:58 -0700 Subject: [PATCH 068/107] Internal changes to versions PiperOrigin-RevId: 655291212 --- editions/golden/compare_cpp_codegen_failure.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editions/golden/compare_cpp_codegen_failure.xml b/editions/golden/compare_cpp_codegen_failure.xml index f992058c2c..cb65ff53a6 100644 --- a/editions/golden/compare_cpp_codegen_failure.xml +++ b/editions/golden/compare_cpp_codegen_failure.xml @@ -2,10 +2,10 @@ - + - + From cdb723815b66e3bac2c9ca87dc9f8322488a6bc9 Mon Sep 17 00:00:00 2001 From: Nelson Liang Date: Tue, 23 Jul 2024 13:56:00 -0700 Subject: [PATCH 069/107] Add Prefetchers to Proto Copy Construct to help address load misses PiperOrigin-RevId: 655292932 --- src/google/protobuf/arena.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index a3cd7b7654..4ec1eaba96 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -29,6 +29,8 @@ using type_info = ::type_info; #include "absl/base/attributes.h" #include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/prefetch.h" #include "absl/log/absl_check.h" #include "absl/utility/internal/if_constexpr.h" #include "google/protobuf/arena_align.h" @@ -665,6 +667,12 @@ PROTOBUF_NOINLINE void* Arena::DefaultConstruct(Arena* arena) { template PROTOBUF_NOINLINE void* Arena::CopyConstruct(Arena* arena, const void* from) { + // If the object is larger than half a cache line, prefetch it. + // This way of prefetching is a little more aggressive than if we + // condition off a whole cache line, but benchmarks show better results. + if (sizeof(T) > ABSL_CACHELINE_SIZE / 2) { + PROTOBUF_PREFETCH_WITH_OFFSET(from, 64); + } static_assert(is_destructor_skippable::value, ""); void* mem = arena != nullptr ? arena->AllocateAligned(sizeof(T)) : ::operator new(sizeof(T)); From 8cdc700b5b2891c6fd91d45a24df6a33188976d9 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Wed, 24 Jul 2024 02:21:54 -0700 Subject: [PATCH 070/107] Implement ctype=CORD for singular string & byte fields - We introduce two new view types ProtoStringCow and ProtoBytesCow. - In UPB, for cord field accessors we always return a Cow::Borrowed. - In C++, for coed field accessors we check if the underlying absl::Cord is flat (contigous) and if so return a Cow::Borrowed. If it's not flat we copy the data to a ProtoString and return a Cow::Owned. - We expect the absl::Cord to be flat almost all the time. We have experimentally verified that for small strings (<4 KiB) and less than 6 appends the cord is in fact flat [1]. - This change lifts the requirement of all ViewProxy types to be Copy. Our Cow types cannot be Copy because the owned types aren't copy. [1] https://source.corp.google.com/piper///depot/google3/experimental/users/buchgr/cords/cords.cc PiperOrigin-RevId: 655485943 --- rust/BUILD | 1 + rust/cord.rs | 117 +++++++ rust/cpp.rs | 6 + rust/proxied.rs | 2 +- rust/shared.rs | 2 + rust/string.rs | 42 +++ rust/test/shared/BUILD | 26 ++ rust/test/shared/ctype_cord_test.rs | 38 +++ rust/test/upb/string_ctypes_test.rs | 8 - .../compiler/rust/accessors/BUILD.bazel | 1 + .../compiler/rust/accessors/accessors.cc | 7 +- .../compiler/rust/accessors/generator.h | 9 + .../compiler/rust/accessors/singular_cord.cc | 288 ++++++++++++++++++ 13 files changed, 536 insertions(+), 11 deletions(-) create mode 100644 rust/cord.rs create mode 100644 rust/test/shared/ctype_cord_test.rs create mode 100644 src/google/protobuf/compiler/rust/accessors/singular_cord.cc diff --git a/rust/BUILD b/rust/BUILD index 2b273172d5..089adeb170 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -52,6 +52,7 @@ PROTOBUF_SHARED = [ "internal.rs", "primitive.rs", "optional.rs", + "cord.rs", "proxied.rs", "repeated.rs", "shared.rs", diff --git a/rust/cord.rs b/rust/cord.rs new file mode 100644 index 0000000000..4715d287a9 --- /dev/null +++ b/rust/cord.rs @@ -0,0 +1,117 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +use crate::__internal::Private; +use crate::{ + AsView, IntoProxied, IntoView, ProtoBytes, ProtoStr, ProtoString, Proxied, Proxy, View, + ViewProxy, +}; +use paste::paste; +use std::cmp::PartialEq; +use std::ops::Deref; + +macro_rules! impl_cord_types { + ($($t:ty, $vt:ty);*) => { + paste! { $( + #[derive(Debug)] + pub struct [< $t Cord>]; + + #[derive(Debug)] + pub enum [< $t Cow>]<'a> { + Borrowed(View<'a, $t>), + Owned($t), + } + + impl Proxied for [< $t Cord>] { + type View<'msg> = [< $t Cow>]<'msg>; + } + + impl<'msg> Proxy<'msg> for [< $t Cow>]<'msg> {} + + impl<'msg> ViewProxy<'msg> for [< $t Cow>]<'msg> {} + + impl<'msg> AsView for [< $t Cow>]<'msg> { + type Proxied = [< $t Cord>]; + + fn as_view(&self) -> [< $t Cow>]<'_> { + match self { + [< $t Cow>]::Owned(owned) => [< $t Cow>]::Borrowed((*owned).as_view()), + [< $t Cow>]::Borrowed(borrowed) => [< $t Cow>]::Borrowed(borrowed), + } + } + } + + impl<'msg> IntoView<'msg> for [< $t Cow>]<'msg> { + fn into_view<'shorter>(self) -> [< $t Cow>]<'shorter> + where + 'msg: 'shorter, { + match self { + [< $t Cow>]::Owned(owned) => [< $t Cow>]::Owned(owned), + [< $t Cow>]::Borrowed(borrow) => [< $t Cow>]::Borrowed(borrow.into_view()), + } + } + } + + impl IntoProxied<$t> for [< $t Cow>]<'_> { + fn into_proxied(self, _private: Private) -> $t { + match self { + [< $t Cow>]::Owned(owned) => owned, + [< $t Cow>]::Borrowed(borrowed) => borrowed.into_proxied(Private), + } + } + } + + impl<'a> Deref for [< $t Cow>]<'a> { + type Target = $vt; + + fn deref(&self) -> View<'_, $t> { + match self { + [< $t Cow>]::Borrowed(borrow) => borrow, + [< $t Cow>]::Owned(owned) => (*owned).as_view(), + } + } + } + + impl AsRef<[u8]> for [< $t Cow>]<'_> { + fn as_ref(&self) -> &[u8] { + match self { + [< $t Cow>]::Borrowed(borrow) => borrow.as_ref(), + [< $t Cow>]::Owned(owned) => owned.as_ref(), + } + } + } + )* + } + } +} + +impl_cord_types!( + ProtoString, ProtoStr; + ProtoBytes, [u8] +); + +macro_rules! impl_eq { + ($($t1:ty, $t2:ty);*) => { + paste! { $( + impl PartialEq<$t1> for $t2 { + fn eq(&self, rhs: &$t1) -> bool { + AsRef::<[u8]>::as_ref(self) == AsRef::<[u8]>::as_ref(rhs) + } + } + )* + } + } +} + +impl_eq!( + ProtoStringCow<'_>, ProtoStringCow<'_>; + str, ProtoStringCow<'_>; + ProtoStringCow<'_>, str; + ProtoBytesCow<'_>, ProtoBytesCow<'_>; + [u8], ProtoBytesCow<'_>; + ProtoBytesCow<'_>, [u8] +); diff --git a/rust/cpp.rs b/rust/cpp.rs index 7d4cf56d63..4410f21d51 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -114,6 +114,12 @@ impl InnerProtoString { let s = ManuallyDrop::new(self); s.owned_ptr } + + /// # Safety + /// - `src` points to a valid CppStdString. + pub unsafe fn from_raw(_private: Private, src: CppStdString) -> InnerProtoString { + InnerProtoString { owned_ptr: src } + } } impl From<&[u8]> for InnerProtoString { diff --git a/rust/proxied.rs b/rust/proxied.rs index 3092109566..b8d327fd3c 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -209,7 +209,7 @@ pub trait IntoMut<'msg>: AsMut { pub trait Proxy<'msg>: 'msg + IntoView<'msg> + Sync + Unpin + Sized + Debug {} /// Declares conversion operations common to view proxies. -pub trait ViewProxy<'msg>: Proxy<'msg> + Copy + Send {} +pub trait ViewProxy<'msg>: Proxy<'msg> + Send {} /// Declares operations common to all mut proxies. /// diff --git a/rust/shared.rs b/rust/shared.rs index 716f34262f..db8655cd6b 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -28,6 +28,7 @@ pub mod __public { write::{Clear, ClearAndParse}, Message, MessageMut, MessageView, }; + pub use crate::cord::{ProtoBytesCow, ProtoStringCow}; pub use crate::r#enum::{Enum, UnknownEnumValue}; pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; pub use crate::optional::Optional; @@ -61,6 +62,7 @@ pub mod __runtime; pub mod __runtime; mod codegen_traits; +mod cord; #[path = "enum.rs"] mod r#enum; mod map; diff --git a/rust/string.rs b/rust/string.rs index 3a90ba3db8..89e6f367aa 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -39,6 +39,15 @@ impl ProtoBytes { pub fn into_inner(self, _private: Private) -> InnerProtoString { self.inner } + + #[doc(hidden)] + pub fn from_inner(_private: Private, inner: InnerProtoString) -> ProtoBytes { + Self { inner } + } + + pub fn as_view(&self) -> &[u8] { + self.inner.as_bytes() + } } impl AsRef<[u8]> for ProtoBytes { @@ -177,6 +186,10 @@ pub struct ProtoString { } impl ProtoString { + pub fn as_view(&self) -> &ProtoStr { + unsafe { ProtoStr::from_utf8_unchecked(self.as_bytes()) } + } + pub fn as_bytes(&self) -> &[u8] { self.inner.as_bytes() } @@ -187,6 +200,17 @@ impl ProtoString { pub fn into_inner(self, _private: Private) -> InnerProtoString { self.inner } + + #[doc(hidden)] + pub fn from_inner(_private: Private, inner: InnerProtoString) -> ProtoString { + Self { inner } + } +} + +impl AsRef<[u8]> for ProtoString { + fn as_ref(&self) -> &[u8] { + self.inner.as_bytes() + } } impl From for ProtoBytes { @@ -539,6 +563,24 @@ impl_bytes_partial_cmp!( <()> str => ProtoStr, ); +impl std::fmt::Debug for ProtoString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + std::fmt::Debug::fmt(self.as_view(), f) + } +} + +impl std::fmt::Debug for ProtoBytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + std::fmt::Debug::fmt(self.as_view(), f) + } +} + +unsafe impl Sync for ProtoString {} +unsafe impl Send for ProtoString {} + +unsafe impl Send for ProtoBytes {} +unsafe impl Sync for ProtoBytes {} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index e39ee34976..01b5610191 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -16,6 +16,32 @@ load("@rules_rust//rust:defs.bzl", "rust_test") +rust_test( + name = "ctype_cord_upb_test", + srcs = ["ctype_cord_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, + deps = [ + "//rust:protobuf_upb_export", + "//rust/test:unittest_upb_rust_proto", + "@crate_index//:googletest", + ], +) + +rust_test( + name = "ctype_cord_cpp_test", + srcs = ["ctype_cord_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, + deps = [ + "//rust:protobuf_cpp_export", + "//rust/test:unittest_cpp_rust_proto", + "@crate_index//:googletest", + ], +) + rust_test( name = "child_parent_upb_test", srcs = ["child_parent_test.rs"], diff --git a/rust/test/shared/ctype_cord_test.rs b/rust/test/shared/ctype_cord_test.rs new file mode 100644 index 0000000000..2879af1906 --- /dev/null +++ b/rust/test/shared/ctype_cord_test.rs @@ -0,0 +1,38 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +use googletest::prelude::*; + +use unittest_rust_proto::{TestAllTypes, TestCord}; + +#[googletest::test] +fn test_string_cord() { + let mut msg = TestAllTypes::new(); + assert_that!(msg.has_optional_cord(), eq(false)); + assert_that!(msg.optional_cord(), eq("")); + msg.set_optional_cord("hello"); + assert_that!(msg.has_optional_cord(), eq(true)); + assert_that!(msg.optional_cord(), eq("hello")); + + let mut msg2 = TestAllTypes::new(); + msg2.set_optional_cord(msg.optional_cord()); + assert_that!(msg2.optional_cord(), eq("hello")); +} + +#[googletest::test] +fn test_bytes_cord() { + let mut msg = TestCord::new(); + assert_that!(msg.has_optional_bytes_cord(), eq(false)); + assert_that!(msg.optional_bytes_cord(), eq("".as_bytes())); + msg.set_optional_bytes_cord(b"hello"); + assert_that!(msg.has_optional_bytes_cord(), eq(true)); + assert_that!(msg.optional_bytes_cord(), eq("hello".as_bytes())); + + let mut msg2 = TestCord::new(); + msg2.set_optional_bytes_cord(msg.optional_bytes_cord()); + assert_that!(msg2.optional_bytes_cord(), eq("hello".as_bytes())); +} diff --git a/rust/test/upb/string_ctypes_test.rs b/rust/test/upb/string_ctypes_test.rs index 8296592b0c..7999a58e45 100644 --- a/rust/test/upb/string_ctypes_test.rs +++ b/rust/test/upb/string_ctypes_test.rs @@ -17,14 +17,6 @@ fn test_stringpiece_repeated() { assert_that!(msg.repeated_string_piece().get(0), some(eq("hello"))); } -#[googletest::test] -fn test_cord() { - let mut msg = TestAllTypes::new(); - assert_that!(msg.optional_cord(), eq("")); - msg.set_optional_cord("hello"); - assert_that!(msg.optional_cord(), eq("hello")); -} - #[googletest::test] fn test_cord_repeated() { let mut msg = TestAllTypes::new(); diff --git a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel index d7bf99e34e..15bc15fab6 100644 --- a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel @@ -13,6 +13,7 @@ cc_library( "default_value.cc", "map.cc", "repeated_field.cc", + "singular_cord.cc", "singular_message.cc", "singular_scalar.cc", "singular_string.cc", diff --git a/src/google/protobuf/compiler/rust/accessors/accessors.cc b/src/google/protobuf/compiler/rust/accessors/accessors.cc index 333260e126..12a5fd72c3 100644 --- a/src/google/protobuf/compiler/rust/accessors/accessors.cc +++ b/src/google/protobuf/compiler/rust/accessors/accessors.cc @@ -30,8 +30,8 @@ std::unique_ptr AccessorGeneratorFor( // ctype=STRING_PIECE fields on cpp kernel yet (upb doesn't care about ctype). auto ctype = field.options().ctype(); if (ctx.is_cpp() && - (ctype == FieldOptions::CORD || - (ctype == FieldOptions::STRING_PIECE && field.is_repeated()))) { + (ctype == FieldOptions::CORD || ctype == FieldOptions::STRING_PIECE) && + field.is_repeated()) { return std::make_unique( "fields has an unsupported ctype"); } @@ -56,6 +56,9 @@ std::unique_ptr AccessorGeneratorFor( return std::make_unique(); case RustFieldType::BYTES: case RustFieldType::STRING: + if (ctype == FieldOptions::CORD) { + return std::make_unique(); + } return std::make_unique(); case RustFieldType::MESSAGE: return std::make_unique(); diff --git a/src/google/protobuf/compiler/rust/accessors/generator.h b/src/google/protobuf/compiler/rust/accessors/generator.h index 8b0cb791de..31761502b2 100644 --- a/src/google/protobuf/compiler/rust/accessors/generator.h +++ b/src/google/protobuf/compiler/rust/accessors/generator.h @@ -94,6 +94,15 @@ class SingularString final : public AccessorGenerator { void InThunkCc(Context& ctx, const FieldDescriptor& field) const override; }; +class SingularCord final : public AccessorGenerator { + public: + ~SingularCord() override = default; + void InMsgImpl(Context& ctx, const FieldDescriptor& field, + AccessorCase accessor_case) const override; + void InExternC(Context& ctx, const FieldDescriptor& field) const override; + void InThunkCc(Context& ctx, const FieldDescriptor& field) const override; +}; + class SingularMessage final : public AccessorGenerator { public: ~SingularMessage() override = default; diff --git a/src/google/protobuf/compiler/rust/accessors/singular_cord.cc b/src/google/protobuf/compiler/rust/accessors/singular_cord.cc new file mode 100644 index 0000000000..900f5b4de3 --- /dev/null +++ b/src/google/protobuf/compiler/rust/accessors/singular_cord.cc @@ -0,0 +1,288 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include + +#include "absl/strings/string_view.h" +#include "google/protobuf/compiler/cpp/helpers.h" +#include "google/protobuf/compiler/rust/accessors/accessor_case.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" +#include "google/protobuf/compiler/rust/context.h" +#include "google/protobuf/compiler/rust/naming.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace rust { + +void SingularCord::InMsgImpl(Context& ctx, const FieldDescriptor& field, + AccessorCase accessor_case) const { + std::string field_name = FieldNameWithCollisionAvoidance(field); + bool is_string_type = field.type() == FieldDescriptor::TYPE_STRING; + ctx.Emit( + {{"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, + {"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"borrowed_getter_thunk", ThunkName(ctx, field, "get_cord_borrowed")}, + {"owned_getter_thunk", ThunkName(ctx, field, "get_cord_owned")}, + {"is_flat_thunk", ThunkName(ctx, field, "cord_is_flat")}, + {"setter_thunk", ThunkName(ctx, field, "set")}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"getter_thunk", ThunkName(ctx, field, "get")}, + {"proxied_type", RsTypePath(ctx, field)}, + {"borrowed_type", + [&] { + if (is_string_type) { + ctx.Emit("$pb$::ProtoStr"); + } else { + ctx.Emit("[u8]"); + } + }}, + {"transform_borrowed", + [&] { + if (is_string_type) { + ctx.Emit(R"rs( + $pb$::ProtoStringCow::Borrowed( + // SAFETY: The runtime doesn't require ProtoStr to be UTF-8. + unsafe { $pb$::ProtoStr::from_utf8_unchecked(view) } + ) + )rs"); + } else { + ctx.Emit(R"rs( + $pb$::ProtoBytesCow::Borrowed( + view + ) + )rs"); + } + }}, + {"transform_owned", + [&] { + if (is_string_type) { + ctx.Emit(R"rs( + $pb$::ProtoStringCow::Owned( + $pb$::ProtoString::from_inner($pbi$::Private, inner) + ) + )rs"); + } else { + ctx.Emit(R"rs( + $pb$::ProtoBytesCow::Owned( + $pb$::ProtoBytes::from_inner($pbi$::Private, inner) + ) + )rs"); + } + }}, + {"view_lifetime", ViewLifetime(accessor_case)}, + {"view_type", + [&] { + if (is_string_type) { + ctx.Emit("$pb$::ProtoStringCow<$view_lifetime$>"); + } else { + ctx.Emit("$pb$::ProtoBytesCow<$view_lifetime$>"); + } + }}, + {"view_self", ViewReceiver(accessor_case)}, + {"getter_impl", + [&] { + if (ctx.is_cpp()) { + ctx.Emit(R"rs( + let cord_is_flat = unsafe { $is_flat_thunk$(self.raw_msg()) }; + if cord_is_flat { + let view = unsafe { $borrowed_getter_thunk$(self.raw_msg()).as_ref() }; + return $transform_borrowed$; + } + + let owned = unsafe { $owned_getter_thunk$(self.raw_msg()) }; + let inner = unsafe { $pbr$::InnerProtoString::from_raw($pbi$::Private, owned) }; + + $transform_owned$ + )rs"); + } else { + ctx.Emit(R"rs( + let view = unsafe { $getter_thunk$(self.raw_msg()).as_ref() }; + $transform_borrowed$ + )rs"); + } + }}, + {"getter", + [&] { + ctx.Emit(R"rs( + pub fn $field$($view_self$) -> $view_type$ { + $getter_impl$ + } + )rs"); + }}, + {"setter_impl", + [&] { + if (ctx.is_cpp()) { + ctx.Emit({}, + R"rs( + let s = val.into_proxied($pbi$::Private); + unsafe { + $setter_thunk$( + self.as_mutator_message_ref($pbi$::Private).msg(), + s.into_inner($pbi$::Private).into_raw($pbi$::Private) + ); + } + )rs"); + } else { + ctx.Emit(R"rs( + let s = val.into_proxied($pbi$::Private); + let (view, arena) = + s.into_inner($pbi$::Private).into_raw_parts($pbi$::Private); + + let mm_ref = + self.as_mutator_message_ref($pbi$::Private); + let parent_arena = mm_ref.arena($pbi$::Private); + + parent_arena.fuse(&arena); + + unsafe { + $setter_thunk$( + self.as_mutator_message_ref($pbi$::Private).msg(), + view + ); + } + )rs"); + } + }}, + {"setter", + [&] { + if (accessor_case == AccessorCase::VIEW) return; + ctx.Emit({}, + R"rs( + pub fn set_$raw_field_name$(&mut self, val: impl $pb$::IntoProxied<$proxied_type$>) { + $setter_impl$ + } + )rs"); + }}, + {"hazzer", + [&] { + if (!field.has_presence()) return; + ctx.Emit({}, R"rs( + pub fn has_$raw_field_name$($view_self$) -> bool { + unsafe { $hazzer_thunk$(self.raw_msg()) } + })rs"); + }}, + {"clearer", + [&] { + if (accessor_case == AccessorCase::VIEW) return; + if (!field.has_presence()) return; + ctx.Emit({}, R"rs( + pub fn clear_$raw_field_name$(&mut self) { + unsafe { $clearer_thunk$(self.raw_msg()) } + })rs"); + }}}, + R"rs( + $getter$ + $setter$ + $hazzer$ + $clearer$ + )rs"); +} + +void SingularCord::InExternC(Context& ctx, const FieldDescriptor& field) const { + ctx.Emit( + {{"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"borrowed_getter_thunk", ThunkName(ctx, field, "get_cord_borrowed")}, + {"owned_getter_thunk", ThunkName(ctx, field, "get_cord_owned")}, + {"is_flat_thunk", ThunkName(ctx, field, "cord_is_flat")}, + {"getter_thunk", ThunkName(ctx, field, "get")}, + {"setter_thunk", ThunkName(ctx, field, "set")}, + {"setter", + [&] { + if (ctx.is_cpp()) { + ctx.Emit(R"rs( + fn $setter_thunk$(raw_msg: $pbr$::RawMessage, val: $pbr$::CppStdString); + )rs"); + } else { + ctx.Emit(R"rs( + fn $setter_thunk$(raw_msg: $pbr$::RawMessage, val: $pbr$::PtrAndLen); + )rs"); + } + }}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"getter_thunks", + [&] { + if (ctx.is_cpp()) { + ctx.Emit(R"rs( + fn $is_flat_thunk$(raw_msg: $pbr$::RawMessage) -> bool; + fn $borrowed_getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::PtrAndLen; + fn $owned_getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::CppStdString; + )rs"); + } else { + ctx.Emit(R"rs( + fn $getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::PtrAndLen; + )rs"); + } + }}, + {"with_presence_fields_thunks", + [&] { + if (field.has_presence()) { + ctx.Emit(R"rs( + fn $hazzer_thunk$(raw_msg: $pbr$::RawMessage) -> bool; + fn $clearer_thunk$(raw_msg: $pbr$::RawMessage); + )rs"); + } + }}}, + R"rs( + $with_presence_fields_thunks$ + $getter_thunks$ + $setter$ + )rs"); +} + +void SingularCord::InThunkCc(Context& ctx, const FieldDescriptor& field) const { + ctx.Emit( + {{"field", cpp::FieldName(&field)}, + {"QualifiedMsg", cpp::QualifiedClassName(field.containing_type())}, + {"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"setter_thunk", ThunkName(ctx, field, "set")}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"borrowed_getter_thunk", ThunkName(ctx, field, "get_cord_borrowed")}, + {"owned_getter_thunk", ThunkName(ctx, field, "get_cord_owned")}, + {"is_flat_thunk", ThunkName(ctx, field, "cord_is_flat")}, + {"with_presence_fields_thunks", + [&] { + if (field.has_presence()) { + ctx.Emit(R"cc( + bool $hazzer_thunk$($QualifiedMsg$* msg) { + return msg->has_$field$(); + } + void $clearer_thunk$($QualifiedMsg$* msg) { msg->clear_$field$(); } + )cc"); + } + }}}, + R"cc( + $with_presence_fields_thunks$; + + bool $is_flat_thunk$($QualifiedMsg$* msg) { + const absl::Cord& cord = msg->$field$(); + return cord.TryFlat().has_value(); + } + ::google::protobuf::rust::PtrAndLen $borrowed_getter_thunk$($QualifiedMsg$* msg) { + const absl::Cord& cord = msg->$field$(); + absl::string_view s = cord.TryFlat().value(); + return ::google::protobuf::rust::PtrAndLen(s.data(), s.size()); + } + std::string* $owned_getter_thunk$($QualifiedMsg$* msg) { + const absl::Cord& cord = msg->$field$(); + std::string* owned = new std::string(); + absl::CopyCordToString(cord, owned); + return owned; + } + void $setter_thunk$($QualifiedMsg$* msg, std::string* s) { + msg->set_$field$(absl::Cord(std::move(*s))); + delete s; + } + )cc"); +} + +} // namespace rust +} // namespace compiler +} // namespace protobuf +} // namespace google From 9f0fc68b38e1082f2e2a81a081fcae3d0abff10e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Jul 2024 09:44:14 +0000 Subject: [PATCH 071/107] Auto-generate files after cl/655485943 --- src/file_lists.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 07152d92d5..fa2eda48e1 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -394,6 +394,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/default_value.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/map.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/repeated_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_cord.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_string.cc From e618b282d44dbf487aeba5da839dd7a0127e9891 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Jul 2024 08:58:57 -0700 Subject: [PATCH 072/107] Add AsView and AsMut implementations to owned messages. Add subtraits to ensure: - All Messages impl AsView + AsMut - All ViewProxies impl AsView - All MutProxies impl AsView + AsMut PiperOrigin-RevId: 655585740 --- rust/codegen_traits.rs | 15 ++++++++------- src/google/protobuf/compiler/rust/message.cc | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index c48c5c02a1..d1a07dea25 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -7,7 +7,7 @@ //! Traits that are implemeted by codegen types. -use crate::{MutProxied, MutProxy, ViewProxy}; +use crate::{AsMut, AsView, MutProxied, MutProxy, ViewProxy}; use create::Parse; use read::Serialize; use std::fmt::Debug; @@ -18,10 +18,10 @@ pub trait Message: MutProxied // Create traits: + create::Parse + Default // Read traits: - + Debug + Serialize + + Debug + Serialize + AsView // Write traits: // TODO: Msg should impl Clear. - + ClearAndParse + + ClearAndParse + AsMut // Thread safety: + Send + Sync // Copy/Clone: @@ -31,11 +31,11 @@ pub trait Message: MutProxied /// A trait that all generated message views implement. pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> // Read traits: - + Debug + Serialize + + Debug + Serialize + AsView // Thread safety: + Send + Sync - // Copy/Clone: - + Copy + Clone + // Copy/Clone: + + Copy + Clone { #[doc(hidden)] type Message: Message; @@ -45,8 +45,9 @@ pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> pub trait MessageMut<'msg>: MutProxy<'msg, MutProxied = Self::Message> // Read traits: - + Debug + Serialize + + Debug + Serialize + AsView // Write traits: + + AsMut // TODO: MsgMut should impl Clear and ClearAndParse. // Thread safety: + Sync diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index b0a16df41c..94f67a7fc2 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -1246,6 +1246,20 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::AsView for $Msg$ { + type Proxied = Self; + fn as_view(&self) -> $Msg$View { + self.as_view() + } + } + + impl $pb$::AsMut for $Msg$ { + type MutProxied = Self; + fn as_mut(&mut self) -> $Msg$Mut { + self.as_mut() + } + } + extern "C" { $Msg_externs$ From 29b755fffd5670738cfb21b20e79d351b72d57b4 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Jul 2024 09:22:56 -0700 Subject: [PATCH 073/107] Skip write of `_cached_size_` when the value to write is `0` and the value was already `0`. This prevents writes to empty messages, and it is required for moving default instances to `rodata`. PiperOrigin-RevId: 655593321 --- src/google/protobuf/message_lite.h | 19 +++++++++++++++++++ src/google/protobuf/wire_format_lite.cc | 14 +++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index c7dfe48e4b..9e462fa5f9 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -88,6 +88,7 @@ class PROTOBUF_EXPORT CachedSize { constexpr CachedSize() noexcept : atom_(Scalar{}) {} // NOLINTNEXTLINE(google-explicit-constructor) constexpr CachedSize(Scalar desired) noexcept : atom_(desired) {} + #if PROTOBUF_BUILTIN_ATOMIC constexpr CachedSize(const CachedSize& other) = default; @@ -96,6 +97,15 @@ class PROTOBUF_EXPORT CachedSize { } void Set(Scalar desired) const noexcept { + // Avoid writing the value when it is zero. This prevents writing to gloabl + // default instances, which might be in readonly memory. + if (ABSL_PREDICT_FALSE(desired == 0)) { + if (Get() == 0) return; + } + __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); + } + + void SetNonZero(Scalar desired) const noexcept { __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); } #else @@ -110,6 +120,15 @@ class PROTOBUF_EXPORT CachedSize { } void Set(Scalar desired) const noexcept { + // Avoid writing the value when it is zero. This prevents writing to gloabl + // default instances, which might be in readonly memory. + if (ABSL_PREDICT_FALSE(desired == 0)) { + if (Get() == 0) return; + } + atom_.store(desired, std::memory_order_relaxed); + } + + void SetNonZero(Scalar desired) const noexcept { atom_.store(desired, std::memory_order_relaxed); } #endif diff --git a/src/google/protobuf/wire_format_lite.cc b/src/google/protobuf/wire_format_lite.cc index e20eda908c..6a9caa9041 100644 --- a/src/google/protobuf/wire_format_lite.cc +++ b/src/google/protobuf/wire_format_lite.cc @@ -840,7 +840,7 @@ size_t WireFormatLite::Int32SizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = Int32Size(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } size_t WireFormatLite::Int64SizeWithPackedTagSize( @@ -852,7 +852,7 @@ size_t WireFormatLite::Int64SizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = Int64Size(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } size_t WireFormatLite::UInt32SizeWithPackedTagSize( @@ -864,7 +864,7 @@ size_t WireFormatLite::UInt32SizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = UInt32Size(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } size_t WireFormatLite::UInt64SizeWithPackedTagSize( @@ -876,7 +876,7 @@ size_t WireFormatLite::UInt64SizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = UInt64Size(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } size_t WireFormatLite::SInt32SizeWithPackedTagSize( @@ -888,7 +888,7 @@ size_t WireFormatLite::SInt32SizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = SInt32Size(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } size_t WireFormatLite::SInt64SizeWithPackedTagSize( @@ -900,7 +900,7 @@ size_t WireFormatLite::SInt64SizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = SInt64Size(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } size_t WireFormatLite::EnumSizeWithPackedTagSize( @@ -912,7 +912,7 @@ size_t WireFormatLite::EnumSizeWithPackedTagSize( } size_t res; PROTOBUF_ALWAYS_INLINE_CALL res = EnumSize(value); - cached_size.Set(ToCachedSize(res)); + cached_size.SetNonZero(ToCachedSize(res)); return tag_size + res + Int32Size(static_cast(res)); } From 165d2c76ed7b98566d12c2bd27353cedf48664f1 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Wed, 24 Jul 2024 09:34:19 -0700 Subject: [PATCH 074/107] Fold 80% of `protos::internal` into `hpb::internal`: ``` - PrivateAccess - GetInternalMsg - CreateMessage - CreateMessageProxy - GetExtensionNumber - ExtensionIdentifier ``` PiperOrigin-RevId: 655597227 --- hpb/hpb.h | 113 +++++++++++++------------- hpb/repeated_field.h | 15 ++-- hpb/repeated_field_iterator.h | 6 +- hpb_generator/gen_accessors.cc | 10 +-- hpb_generator/gen_extensions.cc | 8 +- hpb_generator/gen_messages.cc | 20 ++--- hpb_generator/gen_repeated_fields.cc | 6 +- hpb_generator/tests/test_generated.cc | 12 +-- protos/protos.h | 9 ++ 9 files changed, 106 insertions(+), 93 deletions(-) diff --git a/hpb/hpb.h b/hpb/hpb.h index 73221b05d3..808b9a5bff 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -150,21 +150,7 @@ using EnableIfHpbClass = std::enable_if_t< template using EnableIfMutableProto = std::enable_if_t::value>; -} // namespace internal - -} // namespace hpb - -namespace protos { -using hpb::Arena; -using hpb::ExtensionNotFoundError; -using hpb::MessageAllocationError; -using hpb::MessageDecodeError; -using hpb::MessageEncodeError; -using hpb::Ptr; -using hpb::SourceLocation; -class ExtensionRegistry; -namespace internal { struct PrivateAccess { template static auto* GetInternalMsg(T&& message) { @@ -182,6 +168,7 @@ struct PrivateAccess { static auto CreateMessage(upb_Arena* arena) { return typename T::Proxy(upb_Message_New(T::minitable(), arena), arena); } + template static constexpr uint32_t GetExtensionNumber(const ExtensionId& id) { return id.number(); @@ -245,6 +232,22 @@ class ExtensionIdentifier : public ExtensionMiniTableProvider { friend class PrivateAccess; }; +} // namespace internal + +} // namespace hpb + +namespace protos { +using hpb::Arena; +using hpb::ExtensionNotFoundError; +using hpb::MessageAllocationError; +using hpb::MessageDecodeError; +using hpb::MessageEncodeError; +using hpb::Ptr; +using hpb::SourceLocation; +class ExtensionRegistry; + +namespace internal { + template upb_Arena* GetArena(Ptr message) { return static_cast(message->GetInternalArena()); @@ -303,15 +306,15 @@ template void DeepCopy(Ptr source_message, Ptr target_message) { static_assert(!std::is_const_v); ::protos::internal::DeepCopy( - internal::GetInternalMsg(target_message), - internal::GetInternalMsg(source_message), T::minitable(), + hpb::internal::GetInternalMsg(target_message), + hpb::internal::GetInternalMsg(source_message), T::minitable(), static_cast(target_message->GetInternalArena())); } template typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { - return internal::PrivateAccess::Proxy( - ::protos::internal::DeepClone(internal::GetInternalMsg(message), + return hpb::internal::PrivateAccess::Proxy( + ::protos::internal::DeepClone(hpb::internal::GetInternalMsg(message), T::minitable(), arena), arena); } @@ -338,13 +341,13 @@ template void ClearMessage(hpb::internal::PtrOrRaw message) { auto ptr = Ptr(message); auto minitable = internal::GetMiniTable(ptr); - upb_Message_Clear(internal::GetInternalMsg(ptr), minitable); + upb_Message_Clear(hpb::internal::GetInternalMsg(ptr), minitable); } class ExtensionRegistry { public: ExtensionRegistry( - const std::vector& + const std::vector& extensions, const upb::Arena& arena) : registry_(upb_ExtensionRegistry_New(arena.ptr())) { @@ -370,16 +373,16 @@ template > ABSL_MUST_USE_RESULT bool HasExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { return ::protos::internal::HasExtensionOrUnknown( - ::protos::internal::GetInternalMsg(message), id.mini_table_ext()); + ::hpb::internal::GetInternalMsg(message), id.mini_table_ext()); } template > ABSL_MUST_USE_RESULT bool HasExtension( const T* message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { return HasExtension(Ptr(message), id); } @@ -388,17 +391,16 @@ template > void ClearExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { static_assert(!std::is_const_v, ""); - upb_Message_ClearExtension(internal::GetInternalMsg(message), + upb_Message_ClearExtension(hpb::internal::GetInternalMsg(message), id.mini_table_ext()); } template > void ClearExtension( - T* message, - const ::protos::internal::ExtensionIdentifier& id) { + T* message, const ::hpb::internal::ExtensionIdentifier& id) { ClearExtension(Ptr(message), id); } @@ -407,13 +409,13 @@ template > absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::hpb::internal::ExtensionIdentifier& id, const Extension& value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); - return ::protos::internal::SetExtension(internal::GetInternalMsg(message), - message_arena, id.mini_table_ext(), - internal::GetInternalMsg(&value)); + return ::protos::internal::SetExtension( + hpb::internal::GetInternalMsg(message), message_arena, + id.mini_table_ext(), hpb::internal::GetInternalMsg(&value)); } template > absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::hpb::internal::ExtensionIdentifier& id, Ptr value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); - return ::protos::internal::SetExtension(internal::GetInternalMsg(message), - message_arena, id.mini_table_ext(), - internal::GetInternalMsg(value)); + return ::protos::internal::SetExtension( + hpb::internal::GetInternalMsg(message), message_arena, + id.mini_table_ext(), hpb::internal::GetInternalMsg(value)); } template > absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::hpb::internal::ExtensionIdentifier& id, Extension&& value) { Extension ext = std::move(value); static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); auto* extension_arena = static_cast(ext.GetInternalArena()); return ::protos::internal::MoveExtension( - internal::GetInternalMsg(message), message_arena, id.mini_table_ext(), - internal::GetInternalMsg(&ext), extension_arena); + hpb::internal::GetInternalMsg(message), message_arena, + id.mini_table_ext(), hpb::internal::GetInternalMsg(&ext), + extension_arena); } template > absl::Status SetExtension( - T* message, const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::hpb::internal::ExtensionIdentifier& id, const Extension& value) { return ::protos::SetExtension(Ptr(message), id, value); } @@ -457,7 +460,7 @@ absl::Status SetExtension( template > absl::Status SetExtension( - T* message, const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::hpb::internal::ExtensionIdentifier& id, Extension&& value) { return ::protos::SetExtension(Ptr(message), id, std::forward(value)); @@ -466,7 +469,7 @@ absl::Status SetExtension( template > absl::Status SetExtension( - T* message, const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::hpb::internal::ExtensionIdentifier& id, Ptr value) { return ::protos::SetExtension(Ptr(message), id, value); } @@ -475,17 +478,17 @@ template > absl::StatusOr> GetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { // TODO: Fix const correctness issues. upb_MessageValue value; const bool ok = ::protos::internal::GetOrPromoteExtension( - const_cast(internal::GetInternalMsg(message)), + const_cast(::hpb::internal::GetInternalMsg(message)), id.mini_table_ext(), ::protos::internal::GetArena(message), &value); if (!ok) { return ExtensionNotFoundError( upb_MiniTableExtension_Number(id.mini_table_ext())); } - return Ptr(::protos::internal::CreateMessage( + return Ptr(::hpb::internal::CreateMessage( (upb_Message*)value.msg_val, ::protos::internal::GetArena(message))); } @@ -493,18 +496,18 @@ template > absl::StatusOr> GetExtension( const T* message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { return GetExtension(Ptr(message), id); } template ABSL_MUST_USE_RESULT bool Parse(Ptr message, absl::string_view bytes) { static_assert(!std::is_const_v); - upb_Message_Clear(internal::GetInternalMsg(message), + upb_Message_Clear(::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), - internal::GetInternalMsg(message), + ::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message), /* extreg= */ nullptr, /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; @@ -515,11 +518,11 @@ ABSL_MUST_USE_RESULT bool Parse( Ptr message, absl::string_view bytes, const ::protos::ExtensionRegistry& extension_registry) { static_assert(!std::is_const_v); - upb_Message_Clear(internal::GetInternalMsg(message), + upb_Message_Clear(::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), - internal::GetInternalMsg(message), + ::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message), /* extreg= */ ::protos::internal::GetUpbExtensions(extension_registry), @@ -537,11 +540,11 @@ ABSL_MUST_USE_RESULT bool Parse( template ABSL_MUST_USE_RESULT bool Parse(T* message, absl::string_view bytes) { static_assert(!std::is_const_v); - upb_Message_Clear(internal::GetInternalMsg(message), + upb_Message_Clear(::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), - internal::GetInternalMsg(message), + ::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message), /* extreg= */ nullptr, /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; @@ -582,7 +585,7 @@ template absl::StatusOr Serialize(const T* message, upb::Arena& arena, int options = 0) { return ::protos::internal::Serialize( - internal::GetInternalMsg(message), + ::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message), arena.ptr(), options); } @@ -590,14 +593,14 @@ template absl::StatusOr Serialize(Ptr message, upb::Arena& arena, int options = 0) { return ::protos::internal::Serialize( - internal::GetInternalMsg(message), + ::hpb::internal::GetInternalMsg(message), ::protos::internal::GetMiniTable(message), arena.ptr(), options); } template constexpr uint32_t ExtensionNumber( - internal::ExtensionIdentifier id) { - return internal::PrivateAccess::GetExtensionNumber(id); + ::hpb::internal::ExtensionIdentifier id) { + return ::hpb::internal::PrivateAccess::GetExtensionNumber(id); } } // namespace protos diff --git a/hpb/repeated_field.h b/hpb/repeated_field.h index 396dc3a3ff..04bdee401a 100644 --- a/hpb/repeated_field.h +++ b/hpb/repeated_field.h @@ -100,7 +100,7 @@ class RepeatedFieldProxy // T::CProxy [] operator specialization. typename T::CProxy operator[](size_t n) const { upb_MessageValue message_value = upb_Array_Get(this->arr_, n); - return ::protos::internal::CreateMessage>( + return ::hpb::internal::CreateMessage>( (upb_Message*)message_value.msg_val, this->arena_); } @@ -109,8 +109,8 @@ class RepeatedFieldProxy template > typename T::Proxy operator[](size_t n) { - return ::protos::internal::CreateMessageProxy(this->GetMessage(n), - this->arena_); + return ::hpb::internal::CreateMessageProxy(this->GetMessage(n), + this->arena_); } // Mutable message reference specialization. @@ -119,8 +119,8 @@ class RepeatedFieldProxy void push_back(const T& t) { upb_MessageValue message_value; message_value.msg_val = upb_Message_DeepClone( - PrivateAccess::GetInternalMsg(&t), ::protos::internal::GetMiniTable(&t), - this->arena_); + ::hpb::internal::PrivateAccess::GetInternalMsg(&t), + ::protos::internal::GetMiniTable(&t), this->arena_); upb_Array_Append(this->arr_, message_value, this->arena_); } @@ -129,8 +129,9 @@ class RepeatedFieldProxy typename = std::enable_if_t> void push_back(T&& msg) { upb_MessageValue message_value; - message_value.msg_val = PrivateAccess::GetInternalMsg(&msg); - upb_Arena_Fuse(GetArena(&msg), this->arena_); + message_value.msg_val = + ::hpb::internal::PrivateAccess::GetInternalMsg(&msg); + upb_Arena_Fuse(::protos::internal::GetArena(&msg), this->arena_); upb_Array_Append(this->arr_, message_value, this->arena_); T moved_msg = std::move(msg); } diff --git a/hpb/repeated_field_iterator.h b/hpb/repeated_field_iterator.h index afc05c9949..c6f9f0d46e 100644 --- a/hpb/repeated_field_iterator.h +++ b/hpb/repeated_field_iterator.h @@ -354,10 +354,10 @@ struct MessageIteratorPolicy { void AddOffset(ptrdiff_t offset) { arr += offset; } auto Get() const { if constexpr (std::is_const_v) { - return ::protos::internal::CreateMessage< - typename std::remove_const_t>(*arr, arena); + return ::hpb::internal::CreateMessage>( + *arr, arena); } else { - return ::protos::internal::CreateMessageProxy(*arr, arena); + return ::hpb::internal::CreateMessageProxy(*arr, arena); } } auto Index() const { return arr; } diff --git a/hpb_generator/gen_accessors.cc b/hpb_generator/gen_accessors.cc index d6eb5dac98..0fd646fa2c 100644 --- a/hpb_generator/gen_accessors.cc +++ b/hpb_generator/gen_accessors.cc @@ -244,7 +244,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { if (!has_$2()) { return $4::default_instance(); } - return ::protos::internal::CreateMessage<$4>( + return ::hpb::internal::CreateMessage<$4>( (upb_Message*)($3_$5(msg_)), arena_); } )cc", @@ -256,7 +256,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { output( R"cc( $1 $0::mutable_$2() { - return ::protos::internal::CreateMessageProxy<$4>( + return ::hpb::internal::CreateMessageProxy<$4>( (upb_Message*)($3_mutable_$5(msg_, $6)), $6); } )cc", @@ -295,7 +295,7 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, R"cc( bool $0::set_$1($2 key, $3 value) { upb_Message* clone = upb_Message_DeepClone( - ::protos::internal::PrivateAccess::GetInternalMsg(value), &$9, + ::hpb::internal::PrivateAccess::GetInternalMsg(value), &$9, arena_); $6return $4_$8_set(msg_, $7, ($5*)clone, arena_); } @@ -309,7 +309,7 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, R"cc( bool $0::set_$1($2 key, $3 value) { upb_Message* clone = upb_Message_DeepClone( - ::protos::internal::PrivateAccess::GetInternalMsg(value), &$9, + ::hpb::internal::PrivateAccess::GetInternalMsg(value), &$9, arena_); $6return $4_$8_set(msg_, $7, ($5*)clone, arena_); } @@ -325,7 +325,7 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, $5* msg_value; $7bool success = $4_$9_get(msg_, $8, &msg_value); if (success) { - return ::protos::internal::CreateMessage<$6>(UPB_UPCAST(msg_value), arena_); + return ::hpb::internal::CreateMessage<$6>(UPB_UPCAST(msg_value), arena_); } return absl::NotFoundError(""); } diff --git a/hpb_generator/gen_extensions.cc b/hpb_generator/gen_extensions.cc index 1bfe74b555..f5d44ff3d5 100644 --- a/hpb_generator/gen_extensions.cc +++ b/hpb_generator/gen_extensions.cc @@ -43,16 +43,16 @@ void WriteExtensionIdentifierHeader(const protobuf::FieldDescriptor* ext, if (ext->extension_scope()) { output( R"cc( - static constexpr ::protos::internal::ExtensionIdentifier<$0, $3> $2{ - $4, &$1}; + static constexpr ::hpb::internal::ExtensionIdentifier<$0, $3> $2{$4, + &$1}; )cc", ContainingTypeName(ext), mini_table_name, ext->name(), CppTypeParameterName(ext), ext->number()); } else { output( R"cc( - inline constexpr ::protos::internal::ExtensionIdentifier<$0, $3> $2{ - $4, &$1}; + inline constexpr ::hpb::internal::ExtensionIdentifier<$0, $3> $2{$4, + &$1}; )cc", ContainingTypeName(ext), mini_table_name, ext->name(), CppTypeParameterName(ext), ext->number()); diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index ed0ff87100..a43dc1df96 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -108,7 +108,7 @@ void WriteModelAccessDeclaration(const protobuf::Descriptor* descriptor, friend class $2; friend class $0Proxy; friend class $0CProxy; - friend struct ::protos::internal::PrivateAccess; + friend struct ::hpb::internal::PrivateAccess; $1* msg_; upb_Arena* arena_; )cc", @@ -238,7 +238,7 @@ void WriteModelPublicDeclaration( upb_Arena_Fuse(arena_, arena); } ::hpb::Arena owned_arena_; - friend struct ::protos::internal::PrivateAccess; + friend struct ::hpb::internal::PrivateAccess; friend Proxy; friend CProxy; friend absl::StatusOr<$2>(::protos::Parse<$2>(absl::string_view bytes, @@ -294,9 +294,9 @@ void WriteModelProxyDeclaration(const protobuf::Descriptor* descriptor, $0Proxy(upb_Message* msg, upb_Arena* arena) : internal::$0Access(($1*)msg, arena) {} friend $0::Proxy(::protos::CreateMessage<$0>(::hpb::Arena& arena)); - friend $0::Proxy(::protos::internal::CreateMessageProxy<$0>( - upb_Message*, upb_Arena*)); - friend struct ::protos::internal::PrivateAccess; + friend $0::Proxy(::hpb::internal::CreateMessageProxy<$0>(upb_Message*, + upb_Arena*)); + friend struct ::hpb::internal::PrivateAccess; friend class RepeatedFieldProxy; friend class $0CProxy; friend class $0Access; @@ -347,7 +347,7 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, $0CProxy(const upb_Message* msg, upb_Arena* arena) : internal::$0Access(($1*)msg, arena){}; - friend struct ::protos::internal::PrivateAccess; + friend struct ::hpb::internal::PrivateAccess; friend class RepeatedFieldProxy; friend class ::hpb::Ptr<$0>; friend class ::hpb::Ptr; @@ -393,12 +393,12 @@ void WriteMessageImplementation( $0::$0(const CProxy& from) : $0Access() { arena_ = owned_arena_.ptr(); msg_ = ($1*)::protos::internal::DeepClone( - ::protos::internal::GetInternalMsg(&from), &$2, arena_); + ::hpb::internal::GetInternalMsg(&from), &$2, arena_); } $0::$0(const Proxy& from) : $0(static_cast(from)) {} internal::$0CProxy::$0CProxy($0Proxy m) : $0Access() { arena_ = m.arena_; - msg_ = ($1*)::protos::internal::GetInternalMsg(&m); + msg_ = ($1*)::hpb::internal::GetInternalMsg(&m); } $0& $0::operator=(const $3& from) { arena_ = owned_arena_.ptr(); @@ -408,7 +408,7 @@ void WriteMessageImplementation( $0& $0::operator=(const CProxy& from) { arena_ = owned_arena_.ptr(); msg_ = ($1*)::protos::internal::DeepClone( - ::protos::internal::GetInternalMsg(&from), &$2, arena_); + ::hpb::internal::GetInternalMsg(&from), &$2, arena_); return *this; } )cc", @@ -446,7 +446,7 @@ void WriteMessageImplementation( output( R"cc( ::hpb::Ptr $0::default_instance() { - return ::protos::internal::CreateMessage<$0>( + return ::hpb::internal::CreateMessage<$0>( (upb_Message *)_$0_default_instance_.msg, _$0_default_instance_.arena); } diff --git a/hpb_generator/gen_repeated_fields.cc b/hpb_generator/gen_repeated_fields.cc index 3ea3014ddb..1b4eef3c5a 100644 --- a/hpb_generator/gen_repeated_fields.cc +++ b/hpb_generator/gen_repeated_fields.cc @@ -135,7 +135,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_$5(msg_, &len); assert(index < len); - return ::protos::internal::CreateMessage<$4>( + return ::hpb::internal::CreateMessage<$4>( (upb_Message*)*(ptr + index), arena_); } )cc", @@ -149,7 +149,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, if (!new_msg) { return ::protos::MessageAllocationError(); } - return ::protos::internal::CreateMessageProxy<$4>((upb_Message*)new_msg, $5); + return ::hpb::internal::CreateMessageProxy<$4>((upb_Message*)new_msg, $5); } )cc", class_name, MessagePtrConstType(field, /* const */ false), @@ -162,7 +162,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_$6(msg_, &len); assert(index < len); - return ::protos::internal::CreateMessageProxy<$4>( + return ::hpb::internal::CreateMessageProxy<$4>( (upb_Message*)*(ptr + index), $5); } )cc", diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index eb6ca07300..eabcb075b5 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -712,7 +712,7 @@ TEST(CppGeneratedCode, SetExtension) { // Use a nested scope to make sure the arenas are fused correctly. ThemeExtension extension1; extension1.set_ext_name("Hello World"); - prior_message = ::protos::internal::GetInternalMsg(&extension1); + prior_message = ::hpb::internal::GetInternalMsg(&extension1); EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); EXPECT_EQ( true, @@ -721,7 +721,7 @@ TEST(CppGeneratedCode, SetExtension) { EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); - EXPECT_EQ(::protos::internal::GetInternalMsg(*ext), prior_message); + EXPECT_EQ(::hpb::internal::GetInternalMsg(*ext), prior_message); } TEST(CppGeneratedCode, SetExtensionWithPtr) { @@ -734,7 +734,7 @@ TEST(CppGeneratedCode, SetExtensionWithPtr) { ::hpb::Ptr extension1 = ::protos::CreateMessage(arena); extension1->set_ext_name("Hello World"); - prior_message = ::protos::internal::GetInternalMsg(extension1); + prior_message = ::hpb::internal::GetInternalMsg(extension1); EXPECT_EQ(false, ::protos::HasExtension(model, theme)); auto res = ::protos::SetExtension(model, theme, extension1); EXPECT_EQ(true, res.ok()); @@ -742,7 +742,7 @@ TEST(CppGeneratedCode, SetExtensionWithPtr) { EXPECT_EQ(true, ::protos::HasExtension(model, theme)); auto ext = ::protos::GetExtension(model, theme); EXPECT_TRUE(ext.ok()); - EXPECT_NE(::protos::internal::GetInternalMsg(*ext), prior_message); + EXPECT_NE(::hpb::internal::GetInternalMsg(*ext), prior_message); } #ifndef _MSC_VER @@ -776,7 +776,7 @@ TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { ::hpb::Ptr extension1 = ::protos::CreateMessage(arena); extension1->set_ext_name("Hello World"); - prior_message = ::protos::internal::GetInternalMsg(extension1); + prior_message = ::hpb::internal::GetInternalMsg(extension1); EXPECT_EQ(false, ::protos::HasExtension(model, theme)); auto res = ::protos::SetExtension(model, theme, extension1); EXPECT_EQ(true, res.ok()); @@ -784,7 +784,7 @@ TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { EXPECT_EQ(true, ::protos::HasExtension(model, theme)); auto ext = ::protos::GetExtension(model, theme); EXPECT_TRUE(ext.ok()); - EXPECT_NE(::protos::internal::GetInternalMsg(*ext), prior_message); + EXPECT_NE(::hpb::internal::GetInternalMsg(*ext), prior_message); } TEST(CppGeneratedCode, SetExtensionFusingFailureShouldCopy) { diff --git a/protos/protos.h b/protos/protos.h index a3e8380041..aee9659383 100644 --- a/protos/protos.h +++ b/protos/protos.h @@ -8,4 +8,13 @@ #ifndef UPB_PROTOS_PROTOS_H_ #define UPB_PROTOS_PROTOS_H_ #include "google/protobuf/hpb/hpb.h" +namespace protos { +namespace internal { +using hpb::internal::CreateMessage; +using hpb::internal::CreateMessageProxy; +using hpb::internal::ExtensionIdentifier; +using hpb::internal::GetInternalMsg; +using hpb::internal::PrivateAccess; +} // namespace internal +} // namespace protos #endif From 75b66c2cdefeb1e47773fb42857af842e35f434d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Jul 2024 09:45:01 -0700 Subject: [PATCH 075/107] Prepare UnknownFieldSet for replace `std::string` with `absl::string_view`: - Add overloads that take `absl::Cord` and `std::string&&` as inputs, putting the burden in the implementation instead of users. - Add overload that returns `absl::Span` for callers that need higher performance requirements where we can avoid copies altogether. - Hide the APIs that return `std::string*` when the breaking change is enabled (via `-D PROTOBUF_TEMPORARY_ENABLE_STRING_VIEW_RETURN_TYPE`). PiperOrigin-RevId: 655600399 --- src/google/protobuf/arena_unittest.cc | 2 +- src/google/protobuf/descriptor.cc | 7 +- src/google/protobuf/parse_context.cc | 18 ++++ src/google/protobuf/parse_context.h | 10 +++ src/google/protobuf/unknown_field_set.cc | 40 ++++++++- src/google/protobuf/unknown_field_set.h | 48 +++++++++- .../protobuf/unknown_field_set_unittest.cc | 88 +++++++++++++++++-- src/google/protobuf/wire_format.cc | 23 ++++- 8 files changed, 217 insertions(+), 19 deletions(-) diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index 75e1f83d53..7739f419e3 100644 --- a/src/google/protobuf/arena_unittest.cc +++ b/src/google/protobuf/arena_unittest.cc @@ -652,7 +652,7 @@ TEST(ArenaTest, UnknownFields) { arena_message_3->mutable_unknown_fields()->AddVarint(1000, 42); arena_message_3->mutable_unknown_fields()->AddFixed32(1001, 42); arena_message_3->mutable_unknown_fields()->AddFixed64(1002, 42); - arena_message_3->mutable_unknown_fields()->AddLengthDelimited(1003); + arena_message_3->mutable_unknown_fields()->AddLengthDelimited(1003, ""); arena_message_3->mutable_unknown_fields()->DeleteSubrange(0, 2); arena_message_3->mutable_unknown_fields()->DeleteByNumber(1002); arena_message_3->mutable_unknown_fields()->DeleteByNumber(1003); diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 62a2dd58cc..57597b3cf0 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8921,11 +8921,12 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( new UnknownFieldSet()); switch ((*iter)->type()) { case FieldDescriptor::TYPE_MESSAGE: { - std::string* outstr = - parent_unknown_fields->AddLengthDelimited((*iter)->number()); - ABSL_CHECK(unknown_fields->SerializeToString(outstr)) + std::string outstr; + ABSL_CHECK(unknown_fields->SerializeToString(&outstr)) << "Unexpected failure while serializing option submessage " << debug_msg_name << "\"."; + parent_unknown_fields->AddLengthDelimited((*iter)->number(), + std::move(outstr)); break; } diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index 9b78a57b67..16fc669233 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -12,6 +12,7 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "google/protobuf/message_lite.h" #include "google/protobuf/repeated_field.h" #include "google/protobuf/wire_format_lite.h" @@ -265,6 +266,23 @@ const char* EpsCopyInputStream::ReadCordFallback(const char* ptr, int size, return ptr; } +const char* EpsCopyInputStream::ReadCharsFallback(const char* ptr, + absl::Span out) { + char* out_ptr = out.data(); + ptr = AppendSize(ptr, out.size(), [&](const char* p, int s) { + memcpy(out_ptr, p, s); + out_ptr += s; + }); + + // If we had an error, set the leftover memory to make sure we don't leak + // uninit data in the object. + if (ABSL_PREDICT_FALSE(ptr == nullptr)) { + memset(out_ptr, 0xCD, out.data() + out.size() - out_ptr); + } + + return ptr; +} + const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) { zcis_ = zcis; diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index d40e71c1dc..d6991c1e78 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -212,6 +212,15 @@ class PROTOBUF_EXPORT EpsCopyInputStream { return ReadCordFallback(ptr, size, cord); } + PROTOBUF_NODISCARD const char* ReadChars(const char* ptr, + absl::Span out) { + if (out.size() <= static_cast(buffer_end_ + kSlopBytes - ptr)) { + memcpy(out.data(), ptr, out.size()); + return ptr + out.size(); + } + return ReadCharsFallback(ptr, out); + } + template PROTOBUF_NODISCARD const char* ReadRepeatedFixed(const char* ptr, @@ -369,6 +378,7 @@ class PROTOBUF_EXPORT EpsCopyInputStream { const char* AppendStringFallback(const char* ptr, int size, std::string* str); const char* ReadStringFallback(const char* ptr, int size, std::string* str); const char* ReadCordFallback(const char* ptr, int size, absl::Cord* cord); + const char* ReadCharsFallback(const char* ptr, absl::Span out); static bool ParseEndsInSlopRegion(const char* begin, int overrun, int depth); bool StreamNext(const void** data) { bool res = zcis_->Next(data, &size_); diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index ecf66f5e82..062624ad6f 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -11,10 +11,15 @@ #include "google/protobuf/unknown_field_set.h" +#include +#include +#include + #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "google/protobuf/extension_set.h" #include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/io/coded_stream.h" @@ -117,6 +122,36 @@ void UnknownFieldSet::AddFixed64(int number, uint64_t value) { field.data_.fixed64_ = value; } +void UnknownFieldSet::AddLengthDelimited(int number, const absl::Cord& value) { + auto out = AddLengthDelimitedUninitialized(number, value.size()); + for (absl::string_view part : value.Chunks()) { + memcpy(out.data(), part.data(), part.size()); + out.remove_prefix(part.size()); + } +} + +absl::Span UnknownFieldSet::AddLengthDelimitedUninitialized(int number, + size_t size) { + auto& field = *fields_.Add(); + field.number_ = number; + field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); + std::string* str = field.data_.string_value = + Arena::Create(arena()); + absl::strings_internal::STLStringResizeUninitialized(str, size); + return absl::Span(*str); +} + +template +void UnknownFieldSet::AddLengthDelimited(int number, std::string&& value) { + auto& field = *fields_.Add(); + field.number_ = number; + field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); + field.data_.string_value = + Arena::Create(arena(), std::move(value)); +} +template void UnknownFieldSet::AddLengthDelimited(int, std::string&&); + +#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) std::string* UnknownFieldSet::AddLengthDelimited(int number) { auto& field = *fields_.Add(); field.number_ = number; @@ -124,6 +159,7 @@ std::string* UnknownFieldSet::AddLengthDelimited(int number) { field.data_.string_value = Arena::Create(arena()); return field.data_.string_value; } +#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE UnknownFieldSet* UnknownFieldSet::AddGroup(int number) { auto& field = *fields_.Add(); @@ -284,10 +320,10 @@ class UnknownFieldParserHelper { } const char* ParseLengthDelimited(uint32_t num, const char* ptr, ParseContext* ctx) { - std::string* s = unknown_->AddLengthDelimited(num); int size = ReadSize(&ptr); GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); - return ctx->ReadString(ptr, size, s); + return ctx->ReadChars(ptr, + unknown_->AddLengthDelimitedUninitialized(num, size)); } const char* ParseGroup(uint32_t num, const char* ptr, ParseContext* ctx) { return ctx->ParseGroupInlined(ptr, num * 8 + 3, [&](const char* ptr) { diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index e4cdb70a9a..90823ee81a 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -24,6 +24,7 @@ #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "google/protobuf/arena.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" @@ -47,6 +48,8 @@ class InternalMetadata; // metadata_lite.h class WireFormat; // wire_format.h class MessageSetFieldSkipperUsingCord; // extension_set_heavy.cc +class UnknownFieldParserHelper; +struct UnknownFieldSetTestPeer; #if defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) using UFSStringView = absl::string_view; @@ -87,7 +90,13 @@ class PROTOBUF_EXPORT UnknownField { inline void set_fixed32(uint32_t value); inline void set_fixed64(uint64_t value); inline void set_length_delimited(absl::string_view value); + // template to avoid ambiguous overload resolution. + template + inline void set_length_delimited(std::string&& value); + inline void set_length_delimited(const absl::Cord& value); +#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) inline std::string* mutable_length_delimited(); +#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE inline UnknownFieldSet* mutable_group(); inline size_t GetLengthDelimitedSize() const; @@ -191,7 +200,14 @@ class PROTOBUF_EXPORT UnknownFieldSet { void AddFixed32(int number, uint32_t value); void AddFixed64(int number, uint64_t value); void AddLengthDelimited(int number, absl::string_view value); + // template to avoid ambiguous overload resolution. + template + void AddLengthDelimited(int number, std::string&& value); + void AddLengthDelimited(int number, const absl::Cord& value); + +#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) std::string* AddLengthDelimited(int number); +#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE UnknownFieldSet* AddGroup(int number); // Adds an unknown field from another set. @@ -233,6 +249,10 @@ class PROTOBUF_EXPORT UnknownFieldSet { : UnknownFieldSet(arena) {} private: + friend internal::WireFormat; + friend internal::UnknownFieldParserHelper; + friend internal::UnknownFieldSetTestPeer; + using InternalArenaConstructable_ = void; using DestructorSkippable_ = void; @@ -241,6 +261,14 @@ class PROTOBUF_EXPORT UnknownFieldSet { Arena* arena() { return fields_.GetArena(); } + // Returns a buffer of `size` chars for the user to fill in. + // The buffer is potentially uninitialized memory. Failing to write to it + // might lead to undefined behavior when reading it later. + // Prefer the overloads above when possible. Calling this API without + // validating the `size` parameter can lead to unintentional memory usage and + // potential OOM. + absl::Span AddLengthDelimitedUninitialized(int number, size_t size); + void ClearFallback(); void SwapSlow(UnknownFieldSet* other); @@ -275,7 +303,7 @@ inline void WriteVarint(uint32_t num, uint64_t val, UnknownFieldSet* unknown) { } inline void WriteLengthDelimited(uint32_t num, absl::string_view val, UnknownFieldSet* unknown) { - unknown->AddLengthDelimited(num)->assign(val.data(), val.size()); + unknown->AddLengthDelimited(num, val); } PROTOBUF_EXPORT @@ -331,7 +359,10 @@ inline UnknownField* UnknownFieldSet::mutable_field(int index) { inline void UnknownFieldSet::AddLengthDelimited(int number, const absl::string_view value) { - AddLengthDelimited(number)->assign(value.data(), value.size()); + auto field = AddLengthDelimitedUninitialized(number, value.size()); + if (!value.empty()) { + memcpy(field.data(), value.data(), value.size()); + } } inline int UnknownField::number() const { return static_cast(number_); } @@ -376,10 +407,21 @@ inline void UnknownField::set_length_delimited(const absl::string_view value) { assert(type() == TYPE_LENGTH_DELIMITED); data_.string_value->assign(value.data(), value.size()); } +template +inline void UnknownField::set_length_delimited(std::string&& value) { + assert(type() == TYPE_LENGTH_DELIMITED); + *data_.string_value = std::move(value); +} +inline void UnknownField::set_length_delimited(const absl::Cord& value) { + assert(type() == TYPE_LENGTH_DELIMITED); + absl::CopyCordToString(value, data_.string_value); +} +#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) inline std::string* UnknownField::mutable_length_delimited() { assert(type() == TYPE_LENGTH_DELIMITED); return data_.string_value; } +#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE inline UnknownFieldSet* UnknownField::mutable_group() { assert(type() == TYPE_GROUP); return data_.group_; @@ -397,6 +439,8 @@ inline size_t UnknownField::GetLengthDelimitedSize() const { inline void UnknownField::SetType(Type type) { type_ = type; } +extern template void UnknownFieldSet::AddLengthDelimited(int, std::string&&); + namespace internal { // Add specialization of InternalMetadata::Container to provide arena support. diff --git a/src/google/protobuf/unknown_field_set_unittest.cc b/src/google/protobuf/unknown_field_set_unittest.cc index 27923675f1..39c3114e69 100644 --- a/src/google/protobuf/unknown_field_set_unittest.cc +++ b/src/google/protobuf/unknown_field_set_unittest.cc @@ -14,6 +14,7 @@ #include "google/protobuf/unknown_field_set.h" +#include #include #include @@ -39,10 +40,27 @@ namespace google { namespace protobuf { +namespace internal { +struct UnknownFieldSetTestPeer { + static auto AddLengthDelimitedUninitialized(UnknownFieldSet& set, int number, + size_t length) { + return set.AddLengthDelimitedUninitialized(number, length); + } +}; +} // namespace internal using internal::WireFormat; using ::testing::ElementsAre; +template +T UnknownToProto(const UnknownFieldSet& set) { + T message; + std::string serialized_message; + ABSL_CHECK(set.SerializeToString(&serialized_message)); + ABSL_CHECK(message.ParseFromString(serialized_message)); + return message; +} + class UnknownFieldSetTest : public testing::Test { protected: void SetUp() override { @@ -181,7 +199,13 @@ static void PopulateUFS(UnknownFieldSet& set) { node->AddVarint(1, 100); const char* long_str = "This is a very long string, not sso"; node->AddLengthDelimited(2, long_str); + node->AddLengthDelimited(2, std::string(long_str)); + node->AddLengthDelimited(2, absl::Cord(long_str)); + internal::UnknownFieldSetTestPeer::AddLengthDelimitedUninitialized(*node, 2, + 100); +#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) *node->AddLengthDelimited(3) = long_str; +#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE // Test some recursion too. node = node->AddGroup(4); } @@ -643,12 +667,20 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var"; +#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) str = unknown_fields->AddLengthDelimited(1); shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str"; str->assign(sizeof(std::string) + 1, 'x'); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str2"; +#else + std::string fake_str(31, 'a'); + str = &fake_str; + unknown_fields->AddLengthDelimited(1, fake_str); + shadow_vector.Add(); + EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str2"; +#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE group = unknown_fields->AddGroup(1); shadow_vector.Add(); @@ -751,7 +783,14 @@ TEST_F(UnknownFieldSetTest, SerializeToString) { field_set.AddVarint(1, -1); field_set.AddVarint(2, -2); field_set.AddLengthDelimited(44, "str"); - field_set.AddLengthDelimited(44, "byv"); + field_set.AddLengthDelimited(44, std::string("byv")); + field_set.AddLengthDelimited(44, + absl::Cord("this came from cord and is long")); + memcpy(internal::UnknownFieldSetTestPeer::AddLengthDelimitedUninitialized( + field_set, 44, 10) + .data(), + "0123456789", 10); + field_set.AddFixed32(7, 7); field_set.AddFixed64(8, 8); @@ -760,10 +799,8 @@ TEST_F(UnknownFieldSetTest, SerializeToString) { group_field_set = field_set.AddGroup(46); group_field_set->AddVarint(47, 2048); - unittest::TestAllTypes message; - std::string serialized_message; - ASSERT_TRUE(field_set.SerializeToString(&serialized_message)); - ASSERT_TRUE(message.ParseFromString(serialized_message)); + unittest::TestAllTypes message = + UnknownToProto(field_set); EXPECT_EQ(message.optional_int32(), -1); EXPECT_EQ(message.optional_int64(), -2); @@ -771,8 +808,9 @@ TEST_F(UnknownFieldSetTest, SerializeToString) { EXPECT_EQ(message.optional_uint64(), 4); EXPECT_EQ(message.optional_fixed32(), 7); EXPECT_EQ(message.optional_fixed64(), 8); - EXPECT_EQ(message.repeated_string(0), "str"); - EXPECT_EQ(message.repeated_string(1), "byv"); + EXPECT_THAT(message.repeated_string(), + ElementsAre("str", "byv", "this came from cord and is long", + "0123456789")); EXPECT_EQ(message.repeatedgroup(0).a(), 1024); EXPECT_EQ(message.repeatedgroup(1).a(), 2048); } @@ -818,6 +856,42 @@ TEST_F(UnknownFieldSetTest, SerializeToCord_TestPackedTypes) { EXPECT_THAT(message.packed_uint64(), ElementsAre(5, 6, 7)); } +TEST(UnknownFieldTest, SettersOverrideTheDataProperly) { + using T = unittest::TestAllTypes; + UnknownFieldSet set; + set.AddVarint(T::kOptionalInt32FieldNumber, 2); + set.AddFixed32(T::kOptionalFixed32FieldNumber, 3); + set.AddFixed64(T::kOptionalFixed64FieldNumber, 4); + set.AddLengthDelimited(T::kOptionalStringFieldNumber, "5"); + + T message = UnknownToProto(set); + + EXPECT_EQ(message.optional_int32(), 2); + EXPECT_EQ(message.optional_fixed32(), 3); + EXPECT_EQ(message.optional_fixed64(), 4); + EXPECT_EQ(message.optional_string(), "5"); + + set.mutable_field(0)->set_varint(22); + set.mutable_field(1)->set_fixed32(33); + set.mutable_field(2)->set_fixed64(44); + set.mutable_field(3)->set_length_delimited("55"); + + message = UnknownToProto(set); + + EXPECT_EQ(message.optional_int32(), 22); + EXPECT_EQ(message.optional_fixed32(), 33); + EXPECT_EQ(message.optional_fixed64(), 44); + EXPECT_EQ(message.optional_string(), "55"); + + set.mutable_field(3)->set_length_delimited(std::string("555")); + message = UnknownToProto(set); + EXPECT_EQ(message.optional_string(), "555"); + + set.mutable_field(3)->set_length_delimited(absl::Cord("5555")); + message = UnknownToProto(set); + EXPECT_EQ(message.optional_string(), "5555"); +} + } // namespace } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index 4d7fb6707c..b743354541 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -13,6 +13,7 @@ #include #include +#include #include #include "absl/log/absl_check.h" @@ -84,9 +85,21 @@ bool WireFormat::SkipField(io::CodedInputStream* input, uint32_t tag, if (!input->ReadVarint32(&length)) return false; if (unknown_fields == nullptr) { if (!input->Skip(length)) return false; + } else if (length > 1'000'000) { + // If the provided length is too long, use the `std::string` approach, + // which will grow as it reads data instead of allocating all at the + // beginning. This protects against malformed input. + // Any reasonable value here is fine. + std::string str; + if (!input->ReadString(&str, length)) { + return false; + } + unknown_fields->AddLengthDelimited(number, std::move(str)); } else { - if (!input->ReadString(unknown_fields->AddLengthDelimited(number), - length)) { + if (!input->ReadRaw( + unknown_fields->AddLengthDelimitedUninitialized(number, length) + .data(), + length)) { return false; } } @@ -362,8 +375,10 @@ bool WireFormat::SkipMessageSetField(io::CodedInputStream* input, UnknownFieldSet* unknown_fields) { uint32_t length; if (!input->ReadVarint32(&length)) return false; - return input->ReadString(unknown_fields->AddLengthDelimited(field_number), - length); + return input->ReadRaw( + unknown_fields->AddLengthDelimitedUninitialized(field_number, length) + .data(), + length); } bool WireFormat::ParseAndMergeMessageSetField(uint32_t field_number, From e3fa6aac29f72e27f9db73db26fb71f24509ab7c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Jul 2024 11:21:52 -0700 Subject: [PATCH 076/107] In edition 2024, `Enum_Name(value)` functions return `absl::string_view` by default. Added enum feature `enum_name_uses_string_view` to turn back to the old behavior returning `const std::string&`. PiperOrigin-RevId: 655637240 --- editions/generated_files_test.cc | 6 +- src/google/protobuf/compiler/cpp/enum.cc | 18 +- src/google/protobuf/cpp_edition_defaults.h | 2 +- src/google/protobuf/cpp_features.pb.cc | 114 ++++-- src/google/protobuf/cpp_features.pb.h | 73 +++- src/google/protobuf/cpp_features.proto | 11 + src/google/protobuf/descriptor_unittest.cc | 375 ++++++++++-------- src/google/protobuf/lite_unittest.cc | 3 + .../protobuf/unittest_lite_edition_2024.proto | 10 + 9 files changed, 388 insertions(+), 224 deletions(-) create mode 100644 src/google/protobuf/unittest_lite_edition_2024.proto diff --git a/editions/generated_files_test.cc b/editions/generated_files_test.cc index 7a9b261391..22aaa18f8c 100644 --- a/editions/generated_files_test.cc +++ b/editions/generated_files_test.cc @@ -146,7 +146,11 @@ TEST(Generated, EditionDefaults2023InternalFeatures) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING } + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + } )pb")); } diff --git a/src/google/protobuf/compiler/cpp/enum.cc b/src/google/protobuf/compiler/cpp/enum.cc index c40a5dff12..a8010b0f57 100644 --- a/src/google/protobuf/compiler/cpp/enum.cc +++ b/src/google/protobuf/compiler/cpp/enum.cc @@ -24,6 +24,7 @@ #include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" +#include "google/protobuf/compiler/cpp/generator.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/cpp/names.h" #include "google/protobuf/descriptor.h" @@ -49,6 +50,11 @@ absl::flat_hash_map EnumVars( enum_->containing_type() == nullptr ? "" : absl::StrCat(classname, "_")}, {"kMin", absl::StrCat(min->number())}, {"kMax", absl::StrCat(max->number())}, + {"return_type", CppGenerator::GetResolvedSourceFeatures(*enum_) + .GetExtension(::pb::cpp) + .enum_name_uses_string_view() + ? "::absl::string_view" + : "const std::string&"}, }; } @@ -182,7 +188,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { )cc"); } else { p->Emit(R"cc( - const std::string& $Msg_Enum$_Name($Msg_Enum$ value); + $return_type$ $Msg_Enum$_Name($Msg_Enum$ value); )cc"); } @@ -204,7 +210,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { if (should_cache_ || !has_reflection_) { p->Emit({{"static_assert", write_assert}}, R"cc( template - const std::string& $Msg_Enum$_Name(T value) { + $return_type$ $Msg_Enum$_Name(T value) { $static_assert$; return $Msg_Enum$_Name(static_cast<$Msg_Enum$>(value)); } @@ -216,7 +222,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { // pointers, so if the enum values are sparse, it's not worth it. p->Emit(R"cc( template <> - inline const std::string& $Msg_Enum$_Name($Msg_Enum$ value) { + inline $return_type$ $Msg_Enum$_Name($Msg_Enum$ value) { return ::$proto_ns$::internal::NameOfDenseEnum<$Msg_Enum$_descriptor, $kMin$, $kMax$>( static_cast(value)); @@ -226,7 +232,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { } else { p->Emit({{"static_assert", write_assert}}, R"cc( template - const std::string& $Msg_Enum$_Name(T value) { + $return_type$ $Msg_Enum$_Name(T value) { $static_assert$; return ::$proto_ns$::internal::NameOfEnum($Msg_Enum$_descriptor(), value); } @@ -322,7 +328,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* p) const { p->Emit(R"cc( template - static inline const std::string& $Enum$_Name(T value) { + static inline $return_type$ $Enum$_Name(T value) { return $Msg_Enum$_Name(value); } static inline bool $Enum$_Parse(absl::string_view name, $Enum_$* value) { @@ -514,7 +520,7 @@ void EnumGenerator::GenerateMethods(int idx, io::Printer* p) { $entries_by_number$, }; - const std::string& $Msg_Enum$_Name($Msg_Enum$ value) { + $return_type$ $Msg_Enum$_Name($Msg_Enum$ value) { static const bool kDummy = ::$proto_ns$::internal::InitializeEnumStrings( $Msg_Enum$_entries, $Msg_Enum$_entries_by_number, diff --git a/src/google/protobuf/cpp_edition_defaults.h b/src/google/protobuf/cpp_edition_defaults.h index 33cdbab8a3..52e5fd65ae 100644 --- a/src/google/protobuf/cpp_edition_defaults.h +++ b/src/google/protobuf/cpp_edition_defaults.h @@ -5,7 +5,7 @@ // the C++ runtime. This is used for feature resolution under Editions. // NOLINTBEGIN // clang-format off -#define PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS "\n\035\030\204\007\"\003\302>\000*\023\010\001\020\002\030\002 \003(\0010\002\302>\004\010\001\020\003\n\035\030\347\007\"\003\302>\000*\023\010\002\020\001\030\001 \002(\0010\001\302>\004\010\000\020\003\n\035\030\350\007\"\023\010\001\020\001\030\001 \002(\0010\001\302>\004\010\000\020\003*\003\302>\000\n\035\030\351\007\"\023\010\001\020\001\030\001 \002(\0010\001\302>\004\010\000\020\001*\003\302>\000 \346\007(\351\007" +#define PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS "\n\037\030\204\007\"\003\302>\000*\025\010\001\020\002\030\002 \003(\0010\002\302>\006\010\001\020\003\030\000\n\037\030\347\007\"\003\302>\000*\025\010\002\020\001\030\001 \002(\0010\001\302>\006\010\000\020\003\030\000\n\037\030\350\007\"\023\010\001\020\001\030\001 \002(\0010\001\302>\004\010\000\020\003*\005\302>\002\030\000\n\037\030\351\007\"\025\010\001\020\001\030\001 \002(\0010\001\302>\006\010\000\020\001\030\001*\003\302>\000 \346\007(\351\007" // clang-format on // NOLINTEND diff --git a/src/google/protobuf/cpp_features.pb.cc b/src/google/protobuf/cpp_features.pb.cc index e97ea4d6fc..4dea8f4771 100644 --- a/src/google/protobuf/cpp_features.pb.cc +++ b/src/google/protobuf/cpp_features.pb.cc @@ -28,8 +28,9 @@ namespace pb { inline constexpr CppFeatures::Impl_::Impl_( ::_pbi::ConstantInitialized) noexcept : _cached_size_{0}, + string_type_{static_cast< ::pb::CppFeatures_StringType >(0)}, legacy_closed_enum_{false}, - string_type_{static_cast< ::pb::CppFeatures_StringType >(0)} {} + enum_name_uses_string_view_{false} {} template PROTOBUF_CONSTEXPR CppFeatures::CppFeatures(::_pbi::ConstantInitialized) @@ -67,13 +68,15 @@ const ::uint32_t ~0u, // no sizeof(Split) PROTOBUF_FIELD_OFFSET(::pb::CppFeatures, _impl_.legacy_closed_enum_), PROTOBUF_FIELD_OFFSET(::pb::CppFeatures, _impl_.string_type_), - 0, + PROTOBUF_FIELD_OFFSET(::pb::CppFeatures, _impl_.enum_name_uses_string_view_), 1, + 0, + 2, }; static const ::_pbi::MigrationSchema schemas[] ABSL_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - {0, 10, -1, sizeof(::pb::CppFeatures)}, + {0, 11, -1, sizeof(::pb::CppFeatures)}, }; static const ::_pb::Message* const file_default_instances[] = { &::pb::_CppFeatures_default_instance_._instance, @@ -81,7 +84,7 @@ static const ::_pb::Message* const file_default_instances[] = { const char descriptor_table_protodef_google_2fprotobuf_2fcpp_5ffeatures_2eproto[] ABSL_ATTRIBUTE_SECTION_VARIABLE( protodesc_cold) = { "\n\"google/protobuf/cpp_features.proto\022\002pb" - "\032 google/protobuf/descriptor.proto\"\256\003\n\013C" + "\032 google/protobuf/descriptor.proto\"\374\003\n\013C" "ppFeatures\022\373\001\n\022legacy_closed_enum\030\001 \001(\010B" "\336\001\210\001\001\230\001\004\230\001\001\242\001\t\022\004true\030\204\007\242\001\n\022\005false\030\347\007\262\001\270\001" "\010\350\007\020\350\007\032\257\001The legacy closed enum treatmen" @@ -90,11 +93,13 @@ const char descriptor_table_protodef_google_2fprotobuf_2fcpp_5ffeatures_2eproto[ "m type on the enum definitions themselve" "s rather than on fields.\022Z\n\013string_type\030" "\002 \001(\0162\032.pb.CppFeatures.StringTypeB)\210\001\001\230\001" - "\004\230\001\001\242\001\013\022\006STRING\030\204\007\242\001\t\022\004VIEW\030\351\007\262\001\003\010\350\007\"E\n\n" - "StringType\022\027\n\023STRING_TYPE_UNKNOWN\020\000\022\010\n\004V" - "IEW\020\001\022\010\n\004CORD\020\002\022\n\n\006STRING\020\003::\n\003cpp\022\033.goo" - "gle.protobuf.FeatureSet\030\350\007 \001(\0132\017.pb.CppF" - "eatures" + "\004\230\001\001\242\001\013\022\006STRING\030\204\007\242\001\t\022\004VIEW\030\351\007\262\001\003\010\350\007\022L\n\032" + "enum_name_uses_string_view\030\003 \001(\010B(\210\001\002\230\001\006" + "\230\001\001\242\001\n\022\005false\030\204\007\242\001\t\022\004true\030\351\007\262\001\003\010\351\007\"E\n\nSt" + "ringType\022\027\n\023STRING_TYPE_UNKNOWN\020\000\022\010\n\004VIE" + "W\020\001\022\010\n\004CORD\020\002\022\n\n\006STRING\020\003::\n\003cpp\022\033.googl" + "e.protobuf.FeatureSet\030\350\007 \001(\0132\017.pb.CppFea" + "tures" }; static const ::_pbi::DescriptorTable* const descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2eproto_deps[1] = { @@ -104,7 +109,7 @@ static ::absl::once_flag descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2ep PROTOBUF_CONSTINIT const ::_pbi::DescriptorTable descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2eproto = { false, false, - 567, + 645, descriptor_table_protodef_google_2fprotobuf_2fcpp_5ffeatures_2eproto, "google/protobuf/cpp_features.proto", &descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2eproto_once, @@ -172,11 +177,11 @@ inline PROTOBUF_NDEBUG_INLINE CppFeatures::Impl_::Impl_( inline void CppFeatures::SharedCtor(::_pb::Arena* arena) { new (&_impl_) Impl_(internal_visibility(), arena); ::memset(reinterpret_cast(&_impl_) + - offsetof(Impl_, legacy_closed_enum_), + offsetof(Impl_, string_type_), 0, - offsetof(Impl_, string_type_) - - offsetof(Impl_, legacy_closed_enum_) + - sizeof(Impl_::string_type_)); + offsetof(Impl_, enum_name_uses_string_view_) - + offsetof(Impl_, string_type_) + + sizeof(Impl_::enum_name_uses_string_view_)); } CppFeatures::~CppFeatures() { // @@protoc_insertion_point(destructor:pb.CppFeatures) @@ -217,15 +222,15 @@ const ::google::protobuf::MessageLite::ClassData* CppFeatures::GetClassData() co return _class_data_.base(); } PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 -const ::_pbi::TcParseTable<1, 2, 1, 0, 2> CppFeatures::_table_ = { +const ::_pbi::TcParseTable<2, 3, 1, 0, 2> CppFeatures::_table_ = { { PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_._has_bits_), 0, // no _extensions_ - 2, 8, // max_field_number, fast_idx_mask + 3, 24, // max_field_number, fast_idx_mask offsetof(decltype(_table_), field_lookup_table), - 4294967292, // skipmap + 4294967288, // skipmap offsetof(decltype(_table_), field_entries), - 2, // num_field_entries + 3, // num_field_entries 1, // num_aux_entries offsetof(decltype(_table_), aux_entries), _class_data_.base(), @@ -235,21 +240,28 @@ const ::_pbi::TcParseTable<1, 2, 1, 0, 2> CppFeatures::_table_ = { ::_pbi::TcParser::GetTable<::pb::CppFeatures>(), // to_prefetch #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ + {::_pbi::TcParser::MiniParse, {}}, + // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { + {::_pbi::TcParser::SingularVarintNoZag1(), + {8, 1, 0, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_)}}, // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { {::_pbi::TcParser::FastEr0S1, - {16, 1, 3, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_)}}, - // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - {::_pbi::TcParser::SingularVarintNoZag1(), - {8, 0, 0, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_)}}, + {16, 0, 3, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_)}}, + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + {::_pbi::TcParser::SingularVarintNoZag1(), + {24, 2, 0, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.enum_name_uses_string_view_)}}, }}, {{ 65535, 65535 }}, {{ // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_), _Internal::kHasBitsOffset + 0, 0, + {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_), _Internal::kHasBitsOffset + 1, 0, (0 | ::_fl::kFcOptional | ::_fl::kBool)}, // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_), _Internal::kHasBitsOffset + 1, 0, + {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_), _Internal::kHasBitsOffset + 0, 0, (0 | ::_fl::kFcOptional | ::_fl::kEnumRange)}, + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.enum_name_uses_string_view_), _Internal::kHasBitsOffset + 2, 0, + (0 | ::_fl::kFcOptional | ::_fl::kBool)}, }}, {{ {0, 4}, }}, {{ @@ -264,10 +276,10 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { (void) cached_has_bits; cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - ::memset(&_impl_.legacy_closed_enum_, 0, static_cast<::size_t>( - reinterpret_cast(&_impl_.string_type_) - - reinterpret_cast(&_impl_.legacy_closed_enum_)) + sizeof(_impl_.string_type_)); + if (cached_has_bits & 0x00000007u) { + ::memset(&_impl_.string_type_, 0, static_cast<::size_t>( + reinterpret_cast(&_impl_.enum_name_uses_string_view_) - + reinterpret_cast(&_impl_.string_type_)) + sizeof(_impl_.enum_name_uses_string_view_)); } _impl_._has_bits_.Clear(); _internal_metadata_.Clear<::google::protobuf::UnknownFieldSet>(); @@ -290,19 +302,26 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { cached_has_bits = this_._impl_._has_bits_[0]; // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000001u) { + if (cached_has_bits & 0x00000002u) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray( 1, this_._internal_legacy_closed_enum(), target); } // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000002u) { + if (cached_has_bits & 0x00000001u) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 2, this_._internal_string_type(), target); } + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + if (cached_has_bits & 0x00000004u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteBoolToArray( + 3, this_._internal_enum_name_uses_string_view(), target); + } + if (PROTOBUF_PREDICT_FALSE(this_._internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( @@ -328,16 +347,20 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { ::_pbi::Prefetch5LinesFrom7Lines(&this_); cached_has_bits = this_._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000001u) { - total_size += 2; - } + if (cached_has_bits & 0x00000007u) { // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000002u) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this_._internal_string_type()); } + // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { + if (cached_has_bits & 0x00000002u) { + total_size += 2; + } + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + if (cached_has_bits & 0x00000004u) { + total_size += 2; + } } return this_.MaybeComputeUnknownFieldsSize(total_size, &this_._impl_._cached_size_); @@ -352,12 +375,15 @@ void CppFeatures::MergeImpl(::google::protobuf::MessageLite& to_msg, const ::goo (void) cached_has_bits; cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { + if (cached_has_bits & 0x00000007u) { if (cached_has_bits & 0x00000001u) { - _this->_impl_.legacy_closed_enum_ = from._impl_.legacy_closed_enum_; + _this->_impl_.string_type_ = from._impl_.string_type_; } if (cached_has_bits & 0x00000002u) { - _this->_impl_.string_type_ = from._impl_.string_type_; + _this->_impl_.legacy_closed_enum_ = from._impl_.legacy_closed_enum_; + } + if (cached_has_bits & 0x00000004u) { + _this->_impl_.enum_name_uses_string_view_ = from._impl_.enum_name_uses_string_view_; } } _this->_impl_._has_bits_[0] |= cached_has_bits; @@ -377,11 +403,11 @@ void CppFeatures::InternalSwap(CppFeatures* PROTOBUF_RESTRICT other) { _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); ::google::protobuf::internal::memswap< - PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_) - + sizeof(CppFeatures::_impl_.string_type_) - - PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_)>( - reinterpret_cast(&_impl_.legacy_closed_enum_), - reinterpret_cast(&other->_impl_.legacy_closed_enum_)); + PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.enum_name_uses_string_view_) + + sizeof(CppFeatures::_impl_.enum_name_uses_string_view_) + - PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_)>( + reinterpret_cast(&_impl_.string_type_), + reinterpret_cast(&other->_impl_.string_type_)); } ::google::protobuf::Metadata CppFeatures::GetMetadata() const { diff --git a/src/google/protobuf/cpp_features.pb.h b/src/google/protobuf/cpp_features.pb.h index f5653ca1f6..3c09b51fea 100644 --- a/src/google/protobuf/cpp_features.pb.h +++ b/src/google/protobuf/cpp_features.pb.h @@ -261,9 +261,21 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message // accessors ------------------------------------------------------- enum : int { - kLegacyClosedEnumFieldNumber = 1, kStringTypeFieldNumber = 2, + kLegacyClosedEnumFieldNumber = 1, + kEnumNameUsesStringViewFieldNumber = 3, }; + // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { + bool has_string_type() const; + void clear_string_type() ; + ::pb::CppFeatures_StringType string_type() const; + void set_string_type(::pb::CppFeatures_StringType value); + + private: + ::pb::CppFeatures_StringType _internal_string_type() const; + void _internal_set_string_type(::pb::CppFeatures_StringType value); + + public: // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { bool has_legacy_closed_enum() const; void clear_legacy_closed_enum() ; @@ -275,15 +287,15 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message void _internal_set_legacy_closed_enum(bool value); public: - // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - bool has_string_type() const; - void clear_string_type() ; - ::pb::CppFeatures_StringType string_type() const; - void set_string_type(::pb::CppFeatures_StringType value); + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + bool has_enum_name_uses_string_view() const; + void clear_enum_name_uses_string_view() ; + bool enum_name_uses_string_view() const; + void set_enum_name_uses_string_view(bool value); private: - ::pb::CppFeatures_StringType _internal_string_type() const; - void _internal_set_string_type(::pb::CppFeatures_StringType value); + bool _internal_enum_name_uses_string_view() const; + void _internal_set_enum_name_uses_string_view(bool value); public: // @@protoc_insertion_point(class_scope:pb.CppFeatures) @@ -291,7 +303,7 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message class _Internal; friend class ::google::protobuf::internal::TcParser; static const ::google::protobuf::internal::TcParseTable< - 1, 2, 1, + 2, 3, 1, 0, 2> _table_; @@ -312,8 +324,9 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message const CppFeatures& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; ::google::protobuf::internal::CachedSize _cached_size_; - bool legacy_closed_enum_; int string_type_; + bool legacy_closed_enum_; + bool enum_name_uses_string_view_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -343,13 +356,13 @@ PROTOBUF_EXPORT extern ::google::protobuf::internal::ExtensionIdentifier< // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { inline bool CppFeatures::has_legacy_closed_enum() const { - bool value = (_impl_._has_bits_[0] & 0x00000001u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; return value; } inline void CppFeatures::clear_legacy_closed_enum() { ::google::protobuf::internal::TSanWrite(&_impl_); _impl_.legacy_closed_enum_ = false; - _impl_._has_bits_[0] &= ~0x00000001u; + _impl_._has_bits_[0] &= ~0x00000002u; } inline bool CppFeatures::legacy_closed_enum() const { // @@protoc_insertion_point(field_get:pb.CppFeatures.legacy_closed_enum) @@ -357,7 +370,7 @@ inline bool CppFeatures::legacy_closed_enum() const { } inline void CppFeatures::set_legacy_closed_enum(bool value) { _internal_set_legacy_closed_enum(value); - _impl_._has_bits_[0] |= 0x00000001u; + _impl_._has_bits_[0] |= 0x00000002u; // @@protoc_insertion_point(field_set:pb.CppFeatures.legacy_closed_enum) } inline bool CppFeatures::_internal_legacy_closed_enum() const { @@ -371,13 +384,13 @@ inline void CppFeatures::_internal_set_legacy_closed_enum(bool value) { // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { inline bool CppFeatures::has_string_type() const { - bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000001u) != 0; return value; } inline void CppFeatures::clear_string_type() { ::google::protobuf::internal::TSanWrite(&_impl_); _impl_.string_type_ = 0; - _impl_._has_bits_[0] &= ~0x00000002u; + _impl_._has_bits_[0] &= ~0x00000001u; } inline ::pb::CppFeatures_StringType CppFeatures::string_type() const { // @@protoc_insertion_point(field_get:pb.CppFeatures.string_type) @@ -385,7 +398,7 @@ inline ::pb::CppFeatures_StringType CppFeatures::string_type() const { } inline void CppFeatures::set_string_type(::pb::CppFeatures_StringType value) { _internal_set_string_type(value); - _impl_._has_bits_[0] |= 0x00000002u; + _impl_._has_bits_[0] |= 0x00000001u; // @@protoc_insertion_point(field_set:pb.CppFeatures.string_type) } inline ::pb::CppFeatures_StringType CppFeatures::_internal_string_type() const { @@ -398,6 +411,34 @@ inline void CppFeatures::_internal_set_string_type(::pb::CppFeatures_StringType _impl_.string_type_ = value; } +// optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { +inline bool CppFeatures::has_enum_name_uses_string_view() const { + bool value = (_impl_._has_bits_[0] & 0x00000004u) != 0; + return value; +} +inline void CppFeatures::clear_enum_name_uses_string_view() { + ::google::protobuf::internal::TSanWrite(&_impl_); + _impl_.enum_name_uses_string_view_ = false; + _impl_._has_bits_[0] &= ~0x00000004u; +} +inline bool CppFeatures::enum_name_uses_string_view() const { + // @@protoc_insertion_point(field_get:pb.CppFeatures.enum_name_uses_string_view) + return _internal_enum_name_uses_string_view(); +} +inline void CppFeatures::set_enum_name_uses_string_view(bool value) { + _internal_set_enum_name_uses_string_view(value); + _impl_._has_bits_[0] |= 0x00000004u; + // @@protoc_insertion_point(field_set:pb.CppFeatures.enum_name_uses_string_view) +} +inline bool CppFeatures::_internal_enum_name_uses_string_view() const { + ::google::protobuf::internal::TSanRead(&_impl_); + return _impl_.enum_name_uses_string_view_; +} +inline void CppFeatures::_internal_set_enum_name_uses_string_view(bool value) { + ::google::protobuf::internal::TSanWrite(&_impl_); + _impl_.enum_name_uses_string_view_ = value; +} + #ifdef __GNUC__ #pragma GCC diagnostic pop #endif // __GNUC__ diff --git a/src/google/protobuf/cpp_features.proto b/src/google/protobuf/cpp_features.proto index f3d670428b..99f1661303 100644 --- a/src/google/protobuf/cpp_features.proto +++ b/src/google/protobuf/cpp_features.proto @@ -53,4 +53,15 @@ message CppFeatures { edition_defaults = { edition: EDITION_LEGACY, value: "STRING" }, edition_defaults = { edition: EDITION_2024, value: "VIEW" } ]; + + optional bool enum_name_uses_string_view = 3 [ + retention = RETENTION_SOURCE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2024, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "false" }, + edition_defaults = { edition: EDITION_2024, value: "true" } + ]; } diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index f4d3137099..e3360e171d 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -7546,33 +7546,42 @@ TEST_F(FeaturesTest, Proto2Features) { EXPECT_THAT(file->options(), EqualsProto("")); EXPECT_EQ(GetFeatures(file).GetExtension(pb::test).file_feature(), pb::VALUE1); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: CLOSED - repeated_field_encoding: EXPANDED - utf8_validation: NONE - message_encoding: LENGTH_PREFIXED - json_format: LEGACY_BEST_EFFORT - [pb.cpp] { legacy_closed_enum: true string_type: STRING })pb")); - EXPECT_THAT( - GetCoreFeatures(field), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: CLOSED - repeated_field_encoding: EXPANDED - utf8_validation: NONE - message_encoding: LENGTH_PREFIXED - json_format: LEGACY_BEST_EFFORT - [pb.cpp] { legacy_closed_enum: true string_type: STRING })pb")); - EXPECT_THAT( - GetCoreFeatures(group), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: CLOSED - repeated_field_encoding: EXPANDED - utf8_validation: NONE - message_encoding: DELIMITED - json_format: LEGACY_BEST_EFFORT - [pb.cpp] { legacy_closed_enum: true string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: CLOSED + repeated_field_encoding: EXPANDED + utf8_validation: NONE + message_encoding: LENGTH_PREFIXED + json_format: LEGACY_BEST_EFFORT + [pb.cpp] { + legacy_closed_enum: true + string_type: STRING + enum_name_uses_string_view: false + })pb")); + EXPECT_THAT(GetCoreFeatures(field), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: CLOSED + repeated_field_encoding: EXPANDED + utf8_validation: NONE + message_encoding: LENGTH_PREFIXED + json_format: LEGACY_BEST_EFFORT + [pb.cpp] { + legacy_closed_enum: true + string_type: STRING + enum_name_uses_string_view: false + })pb")); + EXPECT_THAT(GetCoreFeatures(group), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: CLOSED + repeated_field_encoding: EXPANDED + utf8_validation: NONE + message_encoding: DELIMITED + json_format: LEGACY_BEST_EFFORT + [pb.cpp] { + legacy_closed_enum: true + string_type: STRING + enum_name_uses_string_view: false + })pb")); EXPECT_TRUE(field->has_presence()); EXPECT_FALSE(field->requires_utf8_validation()); EXPECT_EQ( @@ -7636,24 +7645,30 @@ TEST_F(FeaturesTest, Proto3Features) { EXPECT_THAT(file->options(), EqualsProto("")); EXPECT_EQ(GetFeatures(file).GetExtension(pb::test).file_feature(), pb::VALUE2); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); - EXPECT_THAT( - GetCoreFeatures(field), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); + EXPECT_THAT(GetCoreFeatures(field), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); EXPECT_FALSE(field->has_presence()); EXPECT_FALSE(field->requires_utf8_validation()); EXPECT_EQ( @@ -7829,7 +7844,11 @@ TEST_F(FeaturesTest, Edition2023Defaults) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING } + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + } )pb")); // Since pb::test is registered in the pool, it should end up with defaults in @@ -7908,7 +7927,11 @@ TEST_F(FeaturesTest, Edition2024Defaults) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: VIEW } + [pb.cpp] { + legacy_closed_enum: false + string_type: VIEW + enum_name_uses_string_view: true + } )pb")); // Since pb::test is registered in the pool, it should end up with defaults in @@ -7937,7 +7960,11 @@ TEST_F(FeaturesBaseTest, DefaultEdition2023Defaults) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING } + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + } )pb")); EXPECT_FALSE(GetFeatures(file).HasExtension(pb::test)); } @@ -7954,15 +7981,18 @@ TEST_F(FeaturesTest, ClearsOptions) { } )pb"); EXPECT_THAT(file->options(), EqualsProto("java_package: 'bar'")); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, RestoresOptionsRoundTrip) { @@ -8319,15 +8349,18 @@ TEST_F(FeaturesTest, NoOptions) { name: "foo.proto" syntax: "editions" edition: EDITION_2023 )pb"); EXPECT_EQ(&file->options(), &FileOptions::default_instance()); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, InvalidEdition) { @@ -8349,15 +8382,18 @@ TEST_F(FeaturesTest, FileFeatures) { options { features { field_presence: IMPLICIT } } )pb"); EXPECT_THAT(file->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, FileFeaturesExtension) { @@ -8427,15 +8463,18 @@ TEST_F(FeaturesTest, MessageFeaturesDefault) { )pb"); const Descriptor* message = file->message_type(0); EXPECT_THAT(message->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(message), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(message), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, MessageFeaturesInherit) { @@ -8535,15 +8574,18 @@ TEST_F(FeaturesTest, FieldFeaturesDefault) { )pb"); const FieldDescriptor* field = file->message_type(0)->field(0); EXPECT_THAT(field->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(field), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(field), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, FieldFeaturesInherit) { @@ -9006,15 +9048,18 @@ TEST_F(FeaturesTest, EnumFeaturesDefault) { )pb"); const EnumDescriptor* enm = file->enum_type(0); EXPECT_THAT(enm->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(enm), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(enm), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, EnumFeaturesInherit) { @@ -9118,15 +9163,18 @@ TEST_F(FeaturesTest, EnumValueFeaturesDefault) { )pb"); const EnumValueDescriptor* value = file->enum_type(0)->value(0); EXPECT_THAT(value->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(value), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(value), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, EnumValueFeaturesInherit) { @@ -9214,15 +9262,18 @@ TEST_F(FeaturesTest, OneofFeaturesDefault) { )pb"); const OneofDescriptor* oneof = file->message_type(0)->oneof_decl(0); EXPECT_THAT(oneof->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(oneof), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(oneof), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, OneofFeaturesInherit) { @@ -9319,15 +9370,18 @@ TEST_F(FeaturesTest, ExtensionRangeFeaturesDefault) { const Descriptor::ExtensionRange* range = file->message_type(0)->extension_range(0); EXPECT_THAT(range->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(range), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(range), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, ExtensionRangeFeaturesInherit) { @@ -9409,15 +9463,18 @@ TEST_F(FeaturesTest, ServiceFeaturesDefault) { )pb"); const ServiceDescriptor* service = file->service(0); EXPECT_THAT(service->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(service), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(service), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, ServiceFeaturesInherit) { @@ -9476,15 +9533,18 @@ TEST_F(FeaturesTest, MethodFeaturesDefault) { )pb"); const MethodDescriptor* method = file->service(0)->method(0); EXPECT_THAT(method->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(method), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(method), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, MethodFeaturesInherit) { @@ -10554,15 +10614,18 @@ TEST_F(FeaturesTest, UninterpretedOptions) { } )pb"); EXPECT_THAT(file->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, UninterpretedOptionsMerge) { diff --git a/src/google/protobuf/lite_unittest.cc b/src/google/protobuf/lite_unittest.cc index bd5f5e7926..be4c428a59 100644 --- a/src/google/protobuf/lite_unittest.cc +++ b/src/google/protobuf/lite_unittest.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,7 @@ #include "google/protobuf/unittest_lite.pb.h" #include "google/protobuf/wire_format_lite.h" + // Must be included last #include "google/protobuf/port_def.inc" @@ -1164,6 +1166,7 @@ TYPED_TEST(LiteTest, EnumValueToName) { EXPECT_EQ("", protobuf_unittest::ForeignEnumLite_Name(999)); } + TYPED_TEST(LiteTest, NestedEnumValueToName) { EXPECT_EQ("FOO", protobuf_unittest::TestAllTypesLite::NestedEnum_Name( protobuf_unittest::TestAllTypesLite::FOO)); diff --git a/src/google/protobuf/unittest_lite_edition_2024.proto b/src/google/protobuf/unittest_lite_edition_2024.proto new file mode 100644 index 0000000000..b56f7d026a --- /dev/null +++ b/src/google/protobuf/unittest_lite_edition_2024.proto @@ -0,0 +1,10 @@ +edition = "2024"; + +package protobuf_unittest; + +option optimize_for = LITE_RUNTIME; + +enum EnumNameStringView { + ENUM_NAME_STRING_VIEW_DEFAULT = 0; + ENUM_NAME_STRING_VIEW_ANOTHER_VALUE = 1; +} From e6304eb3a27428370bbcfe6b217caf1f340aa7c5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Jul 2024 12:28:32 -0700 Subject: [PATCH 077/107] Add AsView + AsMut as supertraits of Proxied and MutProxied. PiperOrigin-RevId: 655660782 --- rust/codegen_traits.rs | 22 ++++++++++------------ rust/cord.rs | 9 ++++++++- rust/internal.rs | 1 + rust/map.rs | 16 ++++++++++++++++ rust/proxied.rs | 18 ++++++++++++++++-- rust/repeated.rs | 22 ++++++++++++++++++++++ rust/string.rs | 16 ++++++++++++++++ 7 files changed, 89 insertions(+), 15 deletions(-) diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index d1a07dea25..ddfe3cfd17 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -7,7 +7,7 @@ //! Traits that are implemeted by codegen types. -use crate::{AsMut, AsView, MutProxied, MutProxy, ViewProxy}; +use crate::{MutProxied, MutProxy, ViewProxy}; use create::Parse; use read::Serialize; use std::fmt::Debug; @@ -16,12 +16,12 @@ use write::ClearAndParse; /// A trait that all generated owned message types implement. pub trait Message: MutProxied // Create traits: - + create::Parse + Default + + Parse + Default // Read traits: - + Debug + Serialize + AsView + + Debug + Serialize // Write traits: // TODO: Msg should impl Clear. - + ClearAndParse + AsMut + + ClearAndParse // Thread safety: + Send + Sync // Copy/Clone: @@ -31,23 +31,22 @@ pub trait Message: MutProxied /// A trait that all generated message views implement. pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> // Read traits: - + Debug + Serialize + AsView + + Debug + Serialize // Thread safety: + Send + Sync // Copy/Clone: + Copy + Clone - { - #[doc(hidden)] - type Message: Message; - } +{ + #[doc(hidden)] + type Message: Message; +} /// A trait that all generated message muts implement. pub trait MessageMut<'msg>: MutProxy<'msg, MutProxied = Self::Message> // Read traits: - + Debug + Serialize + AsView + + Debug + Serialize // Write traits: - + AsMut // TODO: MsgMut should impl Clear and ClearAndParse. // Thread safety: + Sync @@ -60,7 +59,6 @@ pub trait MessageMut<'msg>: /// Operations related to constructing a message. Only owned messages implement /// these traits. pub(crate) mod create { - pub trait Parse: Sized { fn parse(serialized: &[u8]) -> Result; } diff --git a/rust/cord.rs b/rust/cord.rs index 4715d287a9..14c9a60991 100644 --- a/rust/cord.rs +++ b/rust/cord.rs @@ -18,7 +18,7 @@ macro_rules! impl_cord_types { ($($t:ty, $vt:ty);*) => { paste! { $( #[derive(Debug)] - pub struct [< $t Cord>]; + pub struct [< $t Cord>](Private); #[derive(Debug)] pub enum [< $t Cow>]<'a> { @@ -30,6 +30,13 @@ macro_rules! impl_cord_types { type View<'msg> = [< $t Cow>]<'msg>; } + impl AsView for [< $t Cord>] { + type Proxied = Self; + fn as_view(&self) -> [< $t Cow>]<'_> { + unimplemented!("Proto Cord should never be constructed"); + } + } + impl<'msg> Proxy<'msg> for [< $t Cow>]<'msg> {} impl<'msg> ViewProxy<'msg> for [< $t Cow>]<'msg> {} diff --git a/rust/internal.rs b/rust/internal.rs index f8118c88d6..c2bdd007c8 100644 --- a/rust/internal.rs +++ b/rust/internal.rs @@ -21,4 +21,5 @@ pub use crate::ProtoStr; pub use crate::__runtime::{PtrAndLen, RawMap, RawMessage, RawRepeatedField}; /// Used to protect internal-only items from being used accidentally. +#[derive(Debug)] pub struct Private; diff --git a/rust/map.rs b/rust/map.rs index 7013226ec7..f136614830 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -109,10 +109,26 @@ impl> Proxied for Map { type View<'msg> = MapView<'msg, K, V> where K: 'msg, V: 'msg; } +impl> AsView for Map { + type Proxied = Self; + + fn as_view(&self) -> MapView<'_, K, V> { + self.as_view() + } +} + impl> MutProxied for Map { type Mut<'msg> = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } +impl> AsMut for Map { + type MutProxied = Self; + + fn as_mut(&mut self) -> MapMut<'_, K, V> { + self.as_mut() + } +} + impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, V> {} impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapView<'msg, K, V> { diff --git a/rust/proxied.rs b/rust/proxied.rs index b8d327fd3c..be941fa3cd 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -52,7 +52,7 @@ use std::fmt::Debug; /// An instance of a `Proxied` can be accessed immutably via `Proxied::View`. /// /// All Protobuf field types implement `Proxied`. -pub trait Proxied: Sized { +pub trait Proxied: AsView + Sized { /// The proxy type that provides shared access to a `T`, like a `&'msg T`. /// /// Most code should use the type alias [`View`]. @@ -67,7 +67,7 @@ pub trait Proxied: Sized { /// and immutably via `MutProxied::View`. /// /// `MutProxied` is implemented by message, map and repeated field types. -pub trait MutProxied: Proxied { +pub trait MutProxied: Proxied + AsMut { /// The proxy type that provides exclusive mutable access to a `T`, like a /// `&'msg mut T`. /// @@ -263,10 +263,24 @@ mod tests { type View<'msg> = MyProxiedView<'msg>; } + impl AsView for MyProxied { + type Proxied = Self; + fn as_view(&self) -> MyProxiedView<'_> { + self.as_view() + } + } + impl MutProxied for MyProxied { type Mut<'msg> = MyProxiedMut<'msg>; } + impl AsMut for MyProxied { + type MutProxied = Self; + fn as_mut(&mut self) -> MyProxiedMut<'_> { + self.as_mut() + } + } + #[derive(Debug, Clone, Copy)] struct MyProxiedView<'msg> { my_proxied_ref: &'msg MyProxied, diff --git a/rust/repeated.rs b/rust/repeated.rs index 0db3f64a96..972209bec2 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -391,6 +391,17 @@ where type View<'msg> = RepeatedView<'msg, T> where Repeated: 'msg; } +impl AsView for Repeated +where + T: ProxiedInRepeated, +{ + type Proxied = Self; + + fn as_view(&self) -> RepeatedView<'_, T> { + self.as_view() + } +} + impl MutProxied for Repeated where T: ProxiedInRepeated, @@ -398,6 +409,17 @@ where type Mut<'msg> = RepeatedMut<'msg, T> where Repeated: 'msg; } +impl AsMut for Repeated +where + T: ProxiedInRepeated, +{ + type MutProxied = Self; + + fn as_mut(&mut self) -> RepeatedMut<'_, T> { + self.as_mut() + } +} + impl<'msg, T> Proxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} impl<'msg, T> AsView for RepeatedView<'msg, T> diff --git a/rust/string.rs b/rust/string.rs index 89e6f367aa..748fe9b71e 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -72,6 +72,14 @@ impl Proxied for ProtoBytes { type View<'msg> = &'msg [u8]; } +impl AsView for ProtoBytes { + type Proxied = Self; + + fn as_view(&self) -> &[u8] { + self.as_view() + } +} + impl IntoProxied for ProtoBytes { fn into_proxied(self, _private: Private) -> ProtoBytes { self @@ -512,6 +520,14 @@ impl Proxied for ProtoString { type View<'msg> = &'msg ProtoStr; } +impl AsView for ProtoString { + type Proxied = Self; + + fn as_view(&self) -> &ProtoStr { + self.as_view() + } +} + impl<'msg> Proxy<'msg> for &'msg ProtoStr {} impl AsView for &ProtoStr { From 35cfde6d74160c3aeb5442e8fabc1fd55358220a Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 24 Jul 2024 12:28:48 -0700 Subject: [PATCH 078/107] Remove `/utf-8` flag added in #14197 We have received several reports in #17036 that the addition of this flag actually broke the use of command argument files with non-ASCII characters in their names. It looks like #14253 ended up fixing the original issue with a different solution anyway. Hopefully this change fixes the issue with non-ASCII characters. PiperOrigin-RevId: 655660885 --- build_defs/cpp_opts.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl index f667a40881..46b60252f6 100644 --- a/build_defs/cpp_opts.bzl +++ b/build_defs/cpp_opts.bzl @@ -15,7 +15,6 @@ COPTS = select({ "/wd4506", # no definition for inline function 'function' "/wd4800", # 'type' : forcing value to bool 'true' or 'false' (performance warning) "/wd4996", # The compiler encountered a deprecated declaration. - "/utf-8", # Set source and execution character sets to UTF-8 ], "//conditions:default": [ "-DHAVE_ZLIB", From d271692f01ef79c6b4eb9d045a1c5e363a6ddc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Wed, 24 Jul 2024 17:09:24 -0700 Subject: [PATCH 079/107] Internal change. PiperOrigin-RevId: 655750162 --- src/google/protobuf/compiler/java/full/message.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/google/protobuf/compiler/java/full/message.cc b/src/google/protobuf/compiler/java/full/message.cc index 9627be0ae2..887dfe2b2b 100644 --- a/src/google/protobuf/compiler/java/full/message.cc +++ b/src/google/protobuf/compiler/java/full/message.cc @@ -447,6 +447,10 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { "}\n" "\n"); } + if (!context_->options().opensource_runtime) { + printer->Print( + "@com.google.protobuf.Internal.ProtoMethodMayReturnNull\n"); + } printer->Print( vars, "public static $oneof_capitalized_name$Case forNumber(int value) {\n" From 1f717248b761e4f37c7d9cb9773c83b9948d9ae9 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Thu, 25 Jul 2024 06:10:52 -0700 Subject: [PATCH 080/107] [ObjC] Collect unknown fields as data Default to collecting unknown fields as data and then create the `GPBUnknownFieldSet` on demand when requested. This should reduce the as the extra objects are created until requested and clears the way to eventually deprecated `GPBUnknownFieldSet` in the future. This also fixes the failing conformance test related to keeping the ordering. PiperOrigin-RevId: 655929316 --- conformance/failure_list_objc.txt | 5 - objectivec/GPBMessage.m | 459 +++++++++++++++--- objectivec/GPBUnknownFieldSet.m | 14 +- .../GPBUnknownFieldSet_PackagePrivate.h | 4 +- 4 files changed, 405 insertions(+), 77 deletions(-) diff --git a/conformance/failure_list_objc.txt b/conformance/failure_list_objc.txt index 0dd4279587..e34501ead7 100644 --- a/conformance/failure_list_objc.txt +++ b/conformance/failure_list_objc.txt @@ -1,7 +1,2 @@ # JSON input or output tests are skipped (in conformance_objc.m) as mobile # platforms don't support JSON wire format to avoid code bloat. - -Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch -Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch -Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch -Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index b0367699dc..652c791ef0 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -5,9 +5,9 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#import -#import "GPBMessage_PackagePrivate.h" +#import "GPBMessage.h" +#import #import #import #import @@ -20,11 +20,18 @@ #import "GPBDictionary_PackagePrivate.h" #import "GPBExtensionInternals.h" #import "GPBExtensionRegistry.h" +#import "GPBMessage_PackagePrivate.h" #import "GPBRootObject_PackagePrivate.h" +#import "GPBUnknownField.h" +#import "GPBUnknownFieldSet.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownFields_PackagePrivate.h" #import "GPBUtilities_PackagePrivate.h" +// TODO: Consider using on other functions to reduce bloat when +// some compiler optimizations are enabled. +#define GPB_NOINLINE __attribute__((noinline)) + // Returns a new instance that was automatically created by |autocreator| for // its field |field|. static GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, GPBMessage *autocreator, @@ -64,7 +71,16 @@ NSString *const GPBMessageExceptionMessageTooLarge = @interface GPBMessage () { @package + // Only one of these two is ever set, GPBUnknownFieldSet is being deprecated and will + // eventually be removed, but because that api support mutation, once the property is + // fetch it must continue to be used so any mutations will be honored in future operations + // on the message. + // Read only operations that access these two/cause things to migration between them should + // be protected with an @synchronized(self) block (that way the code also doesn't have to + // worry about throws). + NSMutableData *unknownFieldData_; GPBUnknownFieldSet *unknownFields_; + NSMutableDictionary *extensionMap_; // Readonly access to autocreatedExtensionMap_ is protected via readOnlyLock_. NSMutableDictionary *autocreatedExtensionMap_; @@ -103,7 +119,6 @@ static id GetOrCreateMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *fiel static id GetMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *field); static NSMutableDictionary *CloneExtensionMap(NSDictionary *extensionMap, NSZone *zone) __attribute__((ns_returns_retained)); -static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self); #if defined(DEBUG) && DEBUG static NSError *MessageError(NSInteger code, NSDictionary *userInfo) { @@ -133,6 +148,178 @@ static NSError *ErrorFromException(NSException *exception) { return error; } +// Helper to encode varints onto the mutable data, the max size need is 10 bytes. +GPB_NOINLINE +static uint8_t *EncodeVarintU64(uint64_t val, uint8_t *ptr) { + do { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + *(ptr++) = byte; + } while (val); + return ptr; +} + +// Helper to encode varints onto the mutable data, the max size need is 5 bytes. +GPB_NOINLINE +static uint8_t *EncodeVarintU32(uint32_t val, uint8_t *ptr) { + do { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + *(ptr++) = byte; + } while (val); + return ptr; +} + +// Helper to encode signed int32 values as varints onto the mutable data, the max size need is 10 +// bytes. +GPB_NOINLINE +static uint8_t *EncodeVarintS32(int32_t val, uint8_t *ptr) { + if (val >= 0) { + return EncodeVarintU32((uint32_t)val, ptr); + } else { + // Must sign-extend + int64_t extended = val; + return EncodeVarintU64((uint64_t)extended, ptr); + } +} + +GPB_NOINLINE +static void AddUnknownFieldVarint32(GPBMessage *self, uint32_t fieldNumber, int32_t value) { + if (self->unknownFields_) { + [self->unknownFields_ mergeVarintField:fieldNumber value:value]; + return; + } + uint8_t buf[20]; + uint8_t *ptr = buf; + ptr = EncodeVarintU32(GPBWireFormatMakeTag(fieldNumber, GPBWireFormatVarint), ptr); + ptr = EncodeVarintS32(value, ptr); + + if (self->unknownFieldData_ == nil) { + self->unknownFieldData_ = [[NSMutableData alloc] initWithCapacity:ptr - buf]; + GPBBecomeVisibleToAutocreator(self); + } + [self->unknownFieldData_ appendBytes:buf length:ptr - buf]; +} + +GPB_NOINLINE +static void AddUnknownFieldLengthDelimited(GPBMessage *self, uint32_t fieldNumber, NSData *value) { + if (self->unknownFields_) { + [self->unknownFields_ mergeLengthDelimited:fieldNumber value:value]; + return; + } + uint8_t buf[20]; + uint8_t *ptr = buf; + ptr = EncodeVarintU32(GPBWireFormatMakeTag(fieldNumber, GPBWireFormatLengthDelimited), ptr); + ptr = EncodeVarintU64((uint64_t)value.length, ptr); + + if (self->unknownFieldData_ == nil) { + self->unknownFieldData_ = [[NSMutableData alloc] initWithCapacity:(ptr - buf) + value.length]; + GPBBecomeVisibleToAutocreator(self); + } + [self->unknownFieldData_ appendBytes:buf length:ptr - buf]; + [self->unknownFieldData_ appendData:value]; +} + +GPB_NOINLINE +static void AddUnknownMessageSetEntry(GPBMessage *self, uint32_t typeId, NSData *value) { + if (self->unknownFields_) { + // Legacy Set does this odd storage for MessageSet. + [self->unknownFields_ mergeLengthDelimited:typeId value:value]; + return; + } + + uint8_t buf[60]; + uint8_t *ptr = buf; + ptr = EncodeVarintU32(GPBWireFormatMessageSetItemTag, ptr); + ptr = EncodeVarintU32(GPBWireFormatMessageSetTypeIdTag, ptr); + ptr = EncodeVarintU32(typeId, ptr); + ptr = EncodeVarintU32(GPBWireFormatMessageSetMessageTag, ptr); + ptr = EncodeVarintU64((uint64_t)value.length, ptr); + uint8_t *split = ptr; + + ptr = EncodeVarintU32(GPBWireFormatMessageSetItemEndTag, ptr); + uint8_t *end = ptr; + + if (self->unknownFieldData_ == nil) { + self->unknownFieldData_ = [[NSMutableData alloc] initWithCapacity:(end - buf) + value.length]; + GPBBecomeVisibleToAutocreator(self); + } + [self->unknownFieldData_ appendBytes:buf length:split - buf]; + [self->unknownFieldData_ appendData:value]; + [self->unknownFieldData_ appendBytes:split length:end - split]; +} + +GPB_NOINLINE +static void ParseUnknownField(GPBMessage *self, uint32_t tag, GPBCodedInputStream *input) { + if (self->unknownFields_) { + if (![self->unknownFields_ mergeFieldFrom:tag input:input]) { + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unexpected end-group tag"); + } + return; + } + + uint8_t buf[20]; + uint8_t *ptr = buf; + ptr = EncodeVarintU32(tag, ptr); // All will need the tag + NSData *bytesToAppend = nil; + + GPBCodedInputStreamState *state = &input->state_; + + switch (GPBWireFormatGetTagWireType(tag)) { + case GPBWireFormatVarint: { + ptr = EncodeVarintU64(GPBCodedInputStreamReadUInt64(state), ptr); + break; + } + case GPBWireFormatFixed64: { + uint64_t value = GPBCodedInputStreamReadFixed64(state); + *(ptr++) = (uint8_t)(value) & 0xFF; + *(ptr++) = (uint8_t)(value >> 8) & 0xFF; + *(ptr++) = (uint8_t)(value >> 16) & 0xFF; + *(ptr++) = (uint8_t)(value >> 24) & 0xFF; + *(ptr++) = (uint8_t)(value >> 32) & 0xFF; + *(ptr++) = (uint8_t)(value >> 40) & 0xFF; + *(ptr++) = (uint8_t)(value >> 48) & 0xFF; + *(ptr++) = (uint8_t)(value >> 56) & 0xFF; + break; + } + case GPBWireFormatLengthDelimited: { + bytesToAppend = GPBCodedInputStreamReadRetainedBytes(state); + ptr = EncodeVarintU64((uint64_t)bytesToAppend.length, ptr); + break; + } + case GPBWireFormatStartGroup: { + bytesToAppend = GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy( + state, GPBWireFormatGetTagFieldNumber(tag)); + break; + } + case GPBWireFormatEndGroup: + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unexpected end-group tag"); + break; + case GPBWireFormatFixed32: { + uint32_t value = GPBCodedInputStreamReadFixed32(state); + *(ptr++) = (uint8_t)(value) & 0xFF; + *(ptr++) = (uint8_t)(value >> 8) & 0xFF; + *(ptr++) = (uint8_t)(value >> 16) & 0xFF; + *(ptr++) = (uint8_t)(value >> 24) & 0xFF; + break; + } + } + + if (self->unknownFieldData_ == nil) { + self->unknownFieldData_ = + [[NSMutableData alloc] initWithCapacity:(ptr - buf) + bytesToAppend.length]; + GPBBecomeVisibleToAutocreator(self); + } + + [self->unknownFieldData_ appendBytes:buf length:ptr - buf]; + if (bytesToAppend) { + [self->unknownFieldData_ appendData:bytesToAppend]; + [bytesToAppend release]; + } +} + static void CheckExtension(GPBMessage *self, GPBExtensionDescriptor *extension) { if (![self isKindOfClass:extension.containingMessageClass]) { [NSException raise:NSInvalidArgumentException @@ -729,8 +916,7 @@ static void DecodeSingleValueFromInputStream(GPBExtensionDescriptor *extension, if (!enumDescriptor.isClosed || enumDescriptor.enumVerifier(val)) { nsValue = [[NSNumber alloc] initWithInt:val]; } else { - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(messageToGetExtension); - [unknownFields mergeVarintField:extension->description_->fieldNumber value:val]; + AddUnknownFieldVarint32(messageToGetExtension, extension->description_->fieldNumber, val); nsValue = nil; } break; @@ -920,12 +1106,50 @@ void GPBClearMessageAutocreator(GPBMessage *self) { self->autocreatorExtension_ = nil; } -static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { - if (!self->unknownFields_) { - self->unknownFields_ = [[GPBUnknownFieldSet alloc] init]; - GPBBecomeVisibleToAutocreator(self); +GPB_NOINLINE +static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, + GPBUnknownFieldSet *targetSet) { + GPBUnknownFieldSet *unknownFields = targetSet ? targetSet : self->unknownFields_; + +#if defined(DEBUG) && DEBUG + NSCAssert(unknownFields != nil, @"Internal error: unknown fields not initialized."); +#endif + + BOOL isMessageSet = self.descriptor.isWireFormat; + GPBUnknownFieldSet *decodeInto = isMessageSet ? [[GPBUnknownFieldSet alloc] init] : unknownFields; + + GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; + @try { + [decodeInto mergeFromCodedInputStream:input]; + } @catch (NSException *exception) { +#if defined(DEBUG) && DEBUG + NSLog(@"%@: Internal exception while parsing the unknown fields into a Set: %@", [self class], + exception); +#endif + } + [input release]; + + if (isMessageSet) { + // Need to transform the groups back into how Message feeds the data into a MessageSet when + // doing a full MessageSet based decode. + GPBUnknownField *groupField = [decodeInto getField:GPBWireFormatMessageSetItem]; + for (GPBUnknownFieldSet *group in groupField.groupList) { + GPBUnknownField *typeIdField = [group getField:GPBWireFormatMessageSetTypeId]; + GPBUnknownField *messageField = [group getField:GPBWireFormatMessageSetMessage]; + if (typeIdField.varintList.count != 1 || messageField.lengthDelimitedList.count != 1) { +#if defined(DEBUG) && DEBUG + NSCAssert(NO, @"Internal error: MessageSet group missing typeId or message."); +#endif + continue; + } + int32_t fieldNumber = (int32_t)[typeIdField.varintList valueAtIndex:0]; + GPBUnknownField *messageSetField = [[GPBUnknownField alloc] initWithNumber:fieldNumber]; + [messageSetField addLengthDelimited:messageField.lengthDelimitedList[0]]; + [unknownFields addField:messageSetField]; + [messageSetField release]; + } + [decodeInto release]; } - return self->unknownFields_; } @implementation GPBMessage @@ -1162,8 +1386,12 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { GPBMessage *result = [[descriptor.messageClass allocWithZone:zone] init]; [self copyFieldsInto:result zone:zone descriptor:descriptor]; - // Make immutable copies of the extra bits. - result->unknownFields_ = [unknownFields_ copyWithZone:zone]; + + @synchronized(self) { + result->unknownFields_ = [unknownFields_ copyWithZone:zone]; + result->unknownFieldData_ = [unknownFieldData_ mutableCopyWithZone:zone]; + } + result->extensionMap_ = CloneExtensionMap(extensionMap_, zone); return result; } @@ -1237,6 +1465,8 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { [extensionMap_ release]; extensionMap_ = nil; + [unknownFieldData_ release]; + unknownFieldData_ = nil; [unknownFields_ release]; unknownFields_ = nil; @@ -1440,11 +1670,20 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { sortedExtensions:sortedExtensions]; } } - if (descriptor.isWireFormat) { - [unknownFields_ writeAsMessageSetTo:output]; - } else { - [unknownFields_ writeToCodedOutputStream:output]; - } + @synchronized(self) { + if (unknownFieldData_) { +#if defined(DEBUG) && DEBUG + NSAssert(unknownFields_ == nil, @"Internal error both unknown states were set"); +#endif + [output writeRawData:unknownFieldData_]; + } else { + if (descriptor.isWireFormat) { + [unknownFields_ writeAsMessageSetTo:output]; + } else { + [unknownFields_ writeToCodedOutputStream:output]; + } + } + } // @synchronized(self) } - (void)writeDelimitedToOutputStream:(NSOutputStream *)output { @@ -2165,11 +2404,25 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { #pragma mark - Unknown Field Support - (GPBUnknownFieldSet *)unknownFields { - return unknownFields_; + @synchronized(self) { + if (unknownFieldData_) { +#if defined(DEBUG) && DEBUG + NSAssert(unknownFields_ == nil, @"Internal error both unknown states were set"); +#endif + unknownFields_ = [[GPBUnknownFieldSet alloc] init]; + MergeUnknownFieldDataIntoFieldSet(self, unknownFieldData_, nil); + [unknownFieldData_ release]; + unknownFieldData_ = nil; + } + return unknownFields_; + } // @synchronized(self) } - (void)setUnknownFields:(GPBUnknownFieldSet *)unknownFields { - if (unknownFields != unknownFields_) { + if (unknownFields != unknownFields_ || unknownFieldData_ != nil) { + // Changing sets or clearing. + [unknownFieldData_ release]; + unknownFieldData_ = nil; [unknownFields_ release]; unknownFields_ = [unknownFields copy]; GPBBecomeVisibleToAutocreator(self); @@ -2252,19 +2505,18 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { } else { // The extension isn't in the registry, but it was well formed, so the whole group structure // get preserved as an unknown field. - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); + // rawBytes was created via a NoCopy, so it can be reusing a // subrange of another NSData that might go out of scope as things // unwind, so a copy is needed to ensure what is saved in the // unknown fields stays valid. NSData *cloned = [NSData dataWithData:rawBytes]; - [unknownFields mergeMessageSetMessage:typeId data:cloned]; + AddUnknownMessageSetEntry(self, typeId, cloned); } } - (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data { - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); - [unknownFields addUnknownMapEntry:fieldNum value:data]; + AddUnknownFieldLengthDelimited(self, fieldNum, data); } #pragma mark - MergeFromCodedInputStream Support @@ -2337,8 +2589,7 @@ static void MergeSingleFieldFromCodedInputStream(GPBMessage *self, GPBFieldDescr if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { GPBSetInt32IvarWithFieldPrivate(self, field, val); } else { - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); - [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; + AddUnknownFieldVarint32(self, GPBFieldNumber(field), val); } } } // switch @@ -2387,8 +2638,7 @@ static void MergeRepeatedPackedFieldFromCodedInputStream(GPBMessage *self, if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { [(GPBEnumArray *)genericArray addRawValue:val]; } else { - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); - [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; + AddUnknownFieldVarint32(self, GPBFieldNumber(field), val); } break; } @@ -2456,8 +2706,7 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { [(GPBEnumArray *)genericArray addRawValue:val]; } else { - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); - [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; + AddUnknownFieldVarint32(self, GPBFieldNumber(field), val); } break; } @@ -2582,11 +2831,7 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } } - GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); - if (![unknownFields mergeFieldFrom:tag input:input]) { - [NSException raise:NSInternalInconsistencyException - format:@"Internal Error: Unable to parse unknown field %u", tag]; - } + ParseUnknownField(self, tag, input); } // while(YES) } @@ -2708,10 +2953,29 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } // for(fields) // Unknown fields. - if (!unknownFields_) { - [self setUnknownFields:other.unknownFields]; + if (unknownFields_) { +#if defined(DEBUG) && DEBUG + NSAssert(unknownFieldData_ == nil, @"Internal error both unknown states were set"); +#endif + @synchronized(other) { + if (other->unknownFields_) { +#if defined(DEBUG) && DEBUG + NSAssert(other->unknownFieldData_ == nil, @"Internal error both unknown states were set"); +#endif + [unknownFields_ mergeUnknownFields:other->unknownFields_]; + } else if (other->unknownFieldData_) { + MergeUnknownFieldDataIntoFieldSet(self, other->unknownFieldData_, nil); + } + } // @synchronized(other) } else { - [unknownFields_ mergeUnknownFields:other.unknownFields]; + NSData *otherData = GPBMessageUnknownFieldsData(other); + if (otherData) { + if (unknownFieldData_) { + [unknownFieldData_ appendData:otherData]; + } else { + unknownFieldData_ = [otherData mutableCopy]; + } + } } // Extensions @@ -2883,15 +3147,72 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } } - // nil and empty are equal - GPBUnknownFieldSet *otherUnknowns = otherMsg->unknownFields_; - if ([unknownFields_ countOfFields] != 0 || [otherUnknowns countOfFields] != 0) { - if (![unknownFields_ isEqual:otherUnknowns]) { - return NO; + // Mutation while another thread is doing read only access is invalid, so the only thing we + // need to guard against is concurrent r/o access, so we can grab the values (and retain them) + // so we have a version to compare against safely incase the second access causes the transform + // between internal states. + GPBUnknownFieldSet *selfUnknownFields; + NSData *selfUnknownFieldData; + @synchronized(self) { + selfUnknownFields = [unknownFields_ retain]; + selfUnknownFieldData = [unknownFieldData_ retain]; + } + GPBUnknownFieldSet *otherUnknownFields; + NSData *otherUnknownFieldData; + @synchronized(otherMsg) { + otherUnknownFields = [otherMsg->unknownFields_ retain]; + otherUnknownFieldData = [otherMsg->unknownFieldData_ retain]; + } +#if defined(DEBUG) && DEBUG && !defined(NS_BLOCK_ASSERTIONS) + if (selfUnknownFields) { + NSAssert(selfUnknownFieldData == nil, @"Internal error both unknown states were set"); + } + if (otherUnknownFields) { + NSAssert(otherUnknownFieldData == nil, @"Internal error both unknown states were set"); + } +#endif + // Since a developer can set the legacy unknownFieldSet, treat nil and empty as the same. + if (selfUnknownFields && selfUnknownFields.countOfFields == 0) { + [selfUnknownFields release]; + selfUnknownFields = nil; + } + if (otherUnknownFields && otherUnknownFields.countOfFields == 0) { + [otherUnknownFields release]; + otherUnknownFields = nil; + } + + BOOL result = YES; + + if (selfUnknownFieldData && otherUnknownFieldData) { + // Both had data, compare it. + result = [selfUnknownFieldData isEqual:otherUnknownFieldData]; + } else if (selfUnknownFields && otherUnknownFields) { + // Both had fields set, compare them. + result = [selfUnknownFields isEqual:otherUnknownFields]; + } else { + // At this point, we're done to one have a set/nothing, and the other having data/nothing. + GPBUnknownFieldSet *theSet = selfUnknownFields ? selfUnknownFields : otherUnknownFields; + NSData *theData = selfUnknownFieldData ? selfUnknownFieldData : otherUnknownFieldData; + if (theSet) { + if (theData) { + GPBUnknownFieldSet *tempSet = [[GPBUnknownFieldSet alloc] init]; + MergeUnknownFieldDataIntoFieldSet(self, theData, tempSet); + result = [tempSet isEqual:theSet]; + [tempSet release]; + } else { + result = NO; + } + } else { + // It was a data/nothing and nothing, so they equal if the other didn't have data. + result = theData == nil; } } - return YES; + [selfUnknownFields release]; + [selfUnknownFieldData release]; + [otherUnknownFields release]; + [otherUnknownFieldData release]; + return result; } // It is very difficult to implement a generic hash for ProtoBuf messages that @@ -3154,11 +3475,20 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } // for(fields) // Add any unknown fields. - if (descriptor.wireFormat) { - result += [unknownFields_ serializedSizeAsMessageSet]; - } else { - result += [unknownFields_ serializedSize]; - } + @synchronized(self) { + if (unknownFieldData_) { +#if defined(DEBUG) && DEBUG + NSAssert(unknownFields_ == nil, @"Internal error both unknown states were set"); +#endif + result += [unknownFieldData_ length]; + } else { + if (descriptor.wireFormat) { + result += [unknownFields_ serializedSizeAsMessageSet]; + } else { + result += [unknownFields_ serializedSize]; + } + } + } // @synchronized(self) // Add any extensions. for (GPBExtensionDescriptor *extension in extensionMap_) { @@ -3686,20 +4016,29 @@ id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { NSData *GPBMessageUnknownFieldsData(GPBMessage *self) { NSData *result = nil; - GPBUnknownFieldSet *unknownFields = self->unknownFields_; - if (unknownFields) { - if (self.descriptor.isWireFormat) { - NSMutableData *mutableData = - [NSMutableData dataWithLength:unknownFields.serializedSizeAsMessageSet]; - GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; - [unknownFields writeAsMessageSetTo:output]; - [output flush]; - [output release]; - result = mutableData; + @synchronized(self) { + GPBUnknownFieldSet *unknownFields = self->unknownFields_; + if (unknownFields) { +#if defined(DEBUG) && DEBUG + NSCAssert(self->unknownFieldData_ == nil, @"Internal error both unknown states were set"); +#endif + if (self.descriptor.isWireFormat) { + NSMutableData *mutableData = + [NSMutableData dataWithLength:unknownFields.serializedSizeAsMessageSet]; + GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; + [unknownFields writeAsMessageSetTo:output]; + [output flush]; + [output release]; + result = mutableData; + } else { + result = [unknownFields data]; + } } else { - result = [unknownFields data]; + // Internally we can borrow it without a copy since this is immediately used by callers + // and multithreaded access with any mutation is not allow on messages. + result = self->unknownFieldData_; } - } + } // @synchronized(self) return result; } diff --git a/objectivec/GPBUnknownFieldSet.m b/objectivec/GPBUnknownFieldSet.m index 10fb80c0c7..6ec70b8925 100644 --- a/objectivec/GPBUnknownFieldSet.m +++ b/objectivec/GPBUnknownFieldSet.m @@ -271,6 +271,11 @@ static void GPBUnknownFieldSetMergeUnknownFields(__unused const void *key, const [[self mutableFieldForNumber:number create:YES] addVarint:value]; } +- (void)mergeLengthDelimited:(int32_t)fieldNum value:(NSData *)value { + checkNumber(fieldNum); + [[self mutableFieldForNumber:fieldNum create:YES] addLengthDelimited:value]; +} + - (BOOL)mergeFieldFrom:(int32_t)tag input:(GPBCodedInputStream *)input { NSAssert(GPBWireFormatIsValidTag(tag), @"Got passed an invalid tag"); int32_t number = GPBWireFormatGetTagFieldNumber(tag); @@ -313,15 +318,6 @@ static void GPBUnknownFieldSetMergeUnknownFields(__unused const void *key, const } } -- (void)mergeMessageSetMessage:(int32_t)number data:(NSData *)messageData { - [[self mutableFieldForNumber:number create:YES] addLengthDelimited:messageData]; -} - -- (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data { - GPBUnknownField *field = [self mutableFieldForNumber:fieldNum create:YES]; - [field addLengthDelimited:data]; -} - - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input { while (YES) { int32_t tag = GPBCodedInputStreamReadTag(&input->state_); diff --git a/objectivec/GPBUnknownFieldSet_PackagePrivate.h b/objectivec/GPBUnknownFieldSet_PackagePrivate.h index 7de20ed0cf..1d5ff50f45 100644 --- a/objectivec/GPBUnknownFieldSet_PackagePrivate.h +++ b/objectivec/GPBUnknownFieldSet_PackagePrivate.h @@ -27,9 +27,7 @@ - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input; - (void)mergeVarintField:(int32_t)number value:(int32_t)value; +- (void)mergeLengthDelimited:(int32_t)number value:(NSData *)value; - (BOOL)mergeFieldFrom:(int32_t)tag input:(GPBCodedInputStream *)input; -- (void)mergeMessageSetMessage:(int32_t)number data:(NSData *)messageData; - -- (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data; @end From f82f4a4d1355e00a7da1b7b902db94aa68f3cbbd Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Jul 2024 06:35:10 -0700 Subject: [PATCH 081/107] Automated rollback of commit 75b66c2cdefeb1e47773fb42857af842e35f434d. PiperOrigin-RevId: 655934886 --- src/google/protobuf/arena_unittest.cc | 2 +- src/google/protobuf/descriptor.cc | 7 +- src/google/protobuf/parse_context.cc | 18 ---- src/google/protobuf/parse_context.h | 10 --- src/google/protobuf/unknown_field_set.cc | 40 +-------- src/google/protobuf/unknown_field_set.h | 48 +--------- .../protobuf/unknown_field_set_unittest.cc | 88 ++----------------- src/google/protobuf/wire_format.cc | 23 +---- 8 files changed, 19 insertions(+), 217 deletions(-) diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index 7739f419e3..75e1f83d53 100644 --- a/src/google/protobuf/arena_unittest.cc +++ b/src/google/protobuf/arena_unittest.cc @@ -652,7 +652,7 @@ TEST(ArenaTest, UnknownFields) { arena_message_3->mutable_unknown_fields()->AddVarint(1000, 42); arena_message_3->mutable_unknown_fields()->AddFixed32(1001, 42); arena_message_3->mutable_unknown_fields()->AddFixed64(1002, 42); - arena_message_3->mutable_unknown_fields()->AddLengthDelimited(1003, ""); + arena_message_3->mutable_unknown_fields()->AddLengthDelimited(1003); arena_message_3->mutable_unknown_fields()->DeleteSubrange(0, 2); arena_message_3->mutable_unknown_fields()->DeleteByNumber(1002); arena_message_3->mutable_unknown_fields()->DeleteByNumber(1003); diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 57597b3cf0..62a2dd58cc 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8921,12 +8921,11 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( new UnknownFieldSet()); switch ((*iter)->type()) { case FieldDescriptor::TYPE_MESSAGE: { - std::string outstr; - ABSL_CHECK(unknown_fields->SerializeToString(&outstr)) + std::string* outstr = + parent_unknown_fields->AddLengthDelimited((*iter)->number()); + ABSL_CHECK(unknown_fields->SerializeToString(outstr)) << "Unexpected failure while serializing option submessage " << debug_msg_name << "\"."; - parent_unknown_fields->AddLengthDelimited((*iter)->number(), - std::move(outstr)); break; } diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index 16fc669233..9b78a57b67 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -12,7 +12,6 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/span.h" #include "google/protobuf/message_lite.h" #include "google/protobuf/repeated_field.h" #include "google/protobuf/wire_format_lite.h" @@ -266,23 +265,6 @@ const char* EpsCopyInputStream::ReadCordFallback(const char* ptr, int size, return ptr; } -const char* EpsCopyInputStream::ReadCharsFallback(const char* ptr, - absl::Span out) { - char* out_ptr = out.data(); - ptr = AppendSize(ptr, out.size(), [&](const char* p, int s) { - memcpy(out_ptr, p, s); - out_ptr += s; - }); - - // If we had an error, set the leftover memory to make sure we don't leak - // uninit data in the object. - if (ABSL_PREDICT_FALSE(ptr == nullptr)) { - memset(out_ptr, 0xCD, out.data() + out.size() - out_ptr); - } - - return ptr; -} - const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) { zcis_ = zcis; diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index d6991c1e78..d40e71c1dc 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -212,15 +212,6 @@ class PROTOBUF_EXPORT EpsCopyInputStream { return ReadCordFallback(ptr, size, cord); } - PROTOBUF_NODISCARD const char* ReadChars(const char* ptr, - absl::Span out) { - if (out.size() <= static_cast(buffer_end_ + kSlopBytes - ptr)) { - memcpy(out.data(), ptr, out.size()); - return ptr + out.size(); - } - return ReadCharsFallback(ptr, out); - } - template PROTOBUF_NODISCARD const char* ReadRepeatedFixed(const char* ptr, @@ -378,7 +369,6 @@ class PROTOBUF_EXPORT EpsCopyInputStream { const char* AppendStringFallback(const char* ptr, int size, std::string* str); const char* ReadStringFallback(const char* ptr, int size, std::string* str); const char* ReadCordFallback(const char* ptr, int size, absl::Cord* cord); - const char* ReadCharsFallback(const char* ptr, absl::Span out); static bool ParseEndsInSlopRegion(const char* begin, int overrun, int depth); bool StreamNext(const void** data) { bool res = zcis_->Next(data, &size_); diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index 062624ad6f..ecf66f5e82 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -11,15 +11,10 @@ #include "google/protobuf/unknown_field_set.h" -#include -#include -#include - #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/string_view.h" -#include "absl/types/span.h" #include "google/protobuf/extension_set.h" #include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/io/coded_stream.h" @@ -122,36 +117,6 @@ void UnknownFieldSet::AddFixed64(int number, uint64_t value) { field.data_.fixed64_ = value; } -void UnknownFieldSet::AddLengthDelimited(int number, const absl::Cord& value) { - auto out = AddLengthDelimitedUninitialized(number, value.size()); - for (absl::string_view part : value.Chunks()) { - memcpy(out.data(), part.data(), part.size()); - out.remove_prefix(part.size()); - } -} - -absl::Span UnknownFieldSet::AddLengthDelimitedUninitialized(int number, - size_t size) { - auto& field = *fields_.Add(); - field.number_ = number; - field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); - std::string* str = field.data_.string_value = - Arena::Create(arena()); - absl::strings_internal::STLStringResizeUninitialized(str, size); - return absl::Span(*str); -} - -template -void UnknownFieldSet::AddLengthDelimited(int number, std::string&& value) { - auto& field = *fields_.Add(); - field.number_ = number; - field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); - field.data_.string_value = - Arena::Create(arena(), std::move(value)); -} -template void UnknownFieldSet::AddLengthDelimited(int, std::string&&); - -#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) std::string* UnknownFieldSet::AddLengthDelimited(int number) { auto& field = *fields_.Add(); field.number_ = number; @@ -159,7 +124,6 @@ std::string* UnknownFieldSet::AddLengthDelimited(int number) { field.data_.string_value = Arena::Create(arena()); return field.data_.string_value; } -#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE UnknownFieldSet* UnknownFieldSet::AddGroup(int number) { auto& field = *fields_.Add(); @@ -320,10 +284,10 @@ class UnknownFieldParserHelper { } const char* ParseLengthDelimited(uint32_t num, const char* ptr, ParseContext* ctx) { + std::string* s = unknown_->AddLengthDelimited(num); int size = ReadSize(&ptr); GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); - return ctx->ReadChars(ptr, - unknown_->AddLengthDelimitedUninitialized(num, size)); + return ctx->ReadString(ptr, size, s); } const char* ParseGroup(uint32_t num, const char* ptr, ParseContext* ctx) { return ctx->ParseGroupInlined(ptr, num * 8 + 3, [&](const char* ptr) { diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index 90823ee81a..e4cdb70a9a 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -24,7 +24,6 @@ #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/span.h" #include "google/protobuf/arena.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" @@ -48,8 +47,6 @@ class InternalMetadata; // metadata_lite.h class WireFormat; // wire_format.h class MessageSetFieldSkipperUsingCord; // extension_set_heavy.cc -class UnknownFieldParserHelper; -struct UnknownFieldSetTestPeer; #if defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) using UFSStringView = absl::string_view; @@ -90,13 +87,7 @@ class PROTOBUF_EXPORT UnknownField { inline void set_fixed32(uint32_t value); inline void set_fixed64(uint64_t value); inline void set_length_delimited(absl::string_view value); - // template to avoid ambiguous overload resolution. - template - inline void set_length_delimited(std::string&& value); - inline void set_length_delimited(const absl::Cord& value); -#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) inline std::string* mutable_length_delimited(); -#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE inline UnknownFieldSet* mutable_group(); inline size_t GetLengthDelimitedSize() const; @@ -200,14 +191,7 @@ class PROTOBUF_EXPORT UnknownFieldSet { void AddFixed32(int number, uint32_t value); void AddFixed64(int number, uint64_t value); void AddLengthDelimited(int number, absl::string_view value); - // template to avoid ambiguous overload resolution. - template - void AddLengthDelimited(int number, std::string&& value); - void AddLengthDelimited(int number, const absl::Cord& value); - -#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) std::string* AddLengthDelimited(int number); -#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE UnknownFieldSet* AddGroup(int number); // Adds an unknown field from another set. @@ -249,10 +233,6 @@ class PROTOBUF_EXPORT UnknownFieldSet { : UnknownFieldSet(arena) {} private: - friend internal::WireFormat; - friend internal::UnknownFieldParserHelper; - friend internal::UnknownFieldSetTestPeer; - using InternalArenaConstructable_ = void; using DestructorSkippable_ = void; @@ -261,14 +241,6 @@ class PROTOBUF_EXPORT UnknownFieldSet { Arena* arena() { return fields_.GetArena(); } - // Returns a buffer of `size` chars for the user to fill in. - // The buffer is potentially uninitialized memory. Failing to write to it - // might lead to undefined behavior when reading it later. - // Prefer the overloads above when possible. Calling this API without - // validating the `size` parameter can lead to unintentional memory usage and - // potential OOM. - absl::Span AddLengthDelimitedUninitialized(int number, size_t size); - void ClearFallback(); void SwapSlow(UnknownFieldSet* other); @@ -303,7 +275,7 @@ inline void WriteVarint(uint32_t num, uint64_t val, UnknownFieldSet* unknown) { } inline void WriteLengthDelimited(uint32_t num, absl::string_view val, UnknownFieldSet* unknown) { - unknown->AddLengthDelimited(num, val); + unknown->AddLengthDelimited(num)->assign(val.data(), val.size()); } PROTOBUF_EXPORT @@ -359,10 +331,7 @@ inline UnknownField* UnknownFieldSet::mutable_field(int index) { inline void UnknownFieldSet::AddLengthDelimited(int number, const absl::string_view value) { - auto field = AddLengthDelimitedUninitialized(number, value.size()); - if (!value.empty()) { - memcpy(field.data(), value.data(), value.size()); - } + AddLengthDelimited(number)->assign(value.data(), value.size()); } inline int UnknownField::number() const { return static_cast(number_); } @@ -407,21 +376,10 @@ inline void UnknownField::set_length_delimited(const absl::string_view value) { assert(type() == TYPE_LENGTH_DELIMITED); data_.string_value->assign(value.data(), value.size()); } -template -inline void UnknownField::set_length_delimited(std::string&& value) { - assert(type() == TYPE_LENGTH_DELIMITED); - *data_.string_value = std::move(value); -} -inline void UnknownField::set_length_delimited(const absl::Cord& value) { - assert(type() == TYPE_LENGTH_DELIMITED); - absl::CopyCordToString(value, data_.string_value); -} -#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) inline std::string* UnknownField::mutable_length_delimited() { assert(type() == TYPE_LENGTH_DELIMITED); return data_.string_value; } -#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE inline UnknownFieldSet* UnknownField::mutable_group() { assert(type() == TYPE_GROUP); return data_.group_; @@ -439,8 +397,6 @@ inline size_t UnknownField::GetLengthDelimitedSize() const { inline void UnknownField::SetType(Type type) { type_ = type; } -extern template void UnknownFieldSet::AddLengthDelimited(int, std::string&&); - namespace internal { // Add specialization of InternalMetadata::Container to provide arena support. diff --git a/src/google/protobuf/unknown_field_set_unittest.cc b/src/google/protobuf/unknown_field_set_unittest.cc index 39c3114e69..27923675f1 100644 --- a/src/google/protobuf/unknown_field_set_unittest.cc +++ b/src/google/protobuf/unknown_field_set_unittest.cc @@ -14,7 +14,6 @@ #include "google/protobuf/unknown_field_set.h" -#include #include #include @@ -40,27 +39,10 @@ namespace google { namespace protobuf { -namespace internal { -struct UnknownFieldSetTestPeer { - static auto AddLengthDelimitedUninitialized(UnknownFieldSet& set, int number, - size_t length) { - return set.AddLengthDelimitedUninitialized(number, length); - } -}; -} // namespace internal using internal::WireFormat; using ::testing::ElementsAre; -template -T UnknownToProto(const UnknownFieldSet& set) { - T message; - std::string serialized_message; - ABSL_CHECK(set.SerializeToString(&serialized_message)); - ABSL_CHECK(message.ParseFromString(serialized_message)); - return message; -} - class UnknownFieldSetTest : public testing::Test { protected: void SetUp() override { @@ -199,13 +181,7 @@ static void PopulateUFS(UnknownFieldSet& set) { node->AddVarint(1, 100); const char* long_str = "This is a very long string, not sso"; node->AddLengthDelimited(2, long_str); - node->AddLengthDelimited(2, std::string(long_str)); - node->AddLengthDelimited(2, absl::Cord(long_str)); - internal::UnknownFieldSetTestPeer::AddLengthDelimitedUninitialized(*node, 2, - 100); -#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) *node->AddLengthDelimited(3) = long_str; -#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE // Test some recursion too. node = node->AddGroup(4); } @@ -667,20 +643,12 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var"; -#if !defined(PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE) str = unknown_fields->AddLengthDelimited(1); shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str"; str->assign(sizeof(std::string) + 1, 'x'); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str2"; -#else - std::string fake_str(31, 'a'); - str = &fake_str; - unknown_fields->AddLengthDelimited(1, fake_str); - shadow_vector.Add(); - EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str2"; -#endif // PROTOBUF_FUTURE_STRING_VIEW_RETURN_TYPE group = unknown_fields->AddGroup(1); shadow_vector.Add(); @@ -783,14 +751,7 @@ TEST_F(UnknownFieldSetTest, SerializeToString) { field_set.AddVarint(1, -1); field_set.AddVarint(2, -2); field_set.AddLengthDelimited(44, "str"); - field_set.AddLengthDelimited(44, std::string("byv")); - field_set.AddLengthDelimited(44, - absl::Cord("this came from cord and is long")); - memcpy(internal::UnknownFieldSetTestPeer::AddLengthDelimitedUninitialized( - field_set, 44, 10) - .data(), - "0123456789", 10); - + field_set.AddLengthDelimited(44, "byv"); field_set.AddFixed32(7, 7); field_set.AddFixed64(8, 8); @@ -799,8 +760,10 @@ TEST_F(UnknownFieldSetTest, SerializeToString) { group_field_set = field_set.AddGroup(46); group_field_set->AddVarint(47, 2048); - unittest::TestAllTypes message = - UnknownToProto(field_set); + unittest::TestAllTypes message; + std::string serialized_message; + ASSERT_TRUE(field_set.SerializeToString(&serialized_message)); + ASSERT_TRUE(message.ParseFromString(serialized_message)); EXPECT_EQ(message.optional_int32(), -1); EXPECT_EQ(message.optional_int64(), -2); @@ -808,9 +771,8 @@ TEST_F(UnknownFieldSetTest, SerializeToString) { EXPECT_EQ(message.optional_uint64(), 4); EXPECT_EQ(message.optional_fixed32(), 7); EXPECT_EQ(message.optional_fixed64(), 8); - EXPECT_THAT(message.repeated_string(), - ElementsAre("str", "byv", "this came from cord and is long", - "0123456789")); + EXPECT_EQ(message.repeated_string(0), "str"); + EXPECT_EQ(message.repeated_string(1), "byv"); EXPECT_EQ(message.repeatedgroup(0).a(), 1024); EXPECT_EQ(message.repeatedgroup(1).a(), 2048); } @@ -856,42 +818,6 @@ TEST_F(UnknownFieldSetTest, SerializeToCord_TestPackedTypes) { EXPECT_THAT(message.packed_uint64(), ElementsAre(5, 6, 7)); } -TEST(UnknownFieldTest, SettersOverrideTheDataProperly) { - using T = unittest::TestAllTypes; - UnknownFieldSet set; - set.AddVarint(T::kOptionalInt32FieldNumber, 2); - set.AddFixed32(T::kOptionalFixed32FieldNumber, 3); - set.AddFixed64(T::kOptionalFixed64FieldNumber, 4); - set.AddLengthDelimited(T::kOptionalStringFieldNumber, "5"); - - T message = UnknownToProto(set); - - EXPECT_EQ(message.optional_int32(), 2); - EXPECT_EQ(message.optional_fixed32(), 3); - EXPECT_EQ(message.optional_fixed64(), 4); - EXPECT_EQ(message.optional_string(), "5"); - - set.mutable_field(0)->set_varint(22); - set.mutable_field(1)->set_fixed32(33); - set.mutable_field(2)->set_fixed64(44); - set.mutable_field(3)->set_length_delimited("55"); - - message = UnknownToProto(set); - - EXPECT_EQ(message.optional_int32(), 22); - EXPECT_EQ(message.optional_fixed32(), 33); - EXPECT_EQ(message.optional_fixed64(), 44); - EXPECT_EQ(message.optional_string(), "55"); - - set.mutable_field(3)->set_length_delimited(std::string("555")); - message = UnknownToProto(set); - EXPECT_EQ(message.optional_string(), "555"); - - set.mutable_field(3)->set_length_delimited(absl::Cord("5555")); - message = UnknownToProto(set); - EXPECT_EQ(message.optional_string(), "5555"); -} - } // namespace } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index b743354541..4d7fb6707c 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -13,7 +13,6 @@ #include #include -#include #include #include "absl/log/absl_check.h" @@ -85,21 +84,9 @@ bool WireFormat::SkipField(io::CodedInputStream* input, uint32_t tag, if (!input->ReadVarint32(&length)) return false; if (unknown_fields == nullptr) { if (!input->Skip(length)) return false; - } else if (length > 1'000'000) { - // If the provided length is too long, use the `std::string` approach, - // which will grow as it reads data instead of allocating all at the - // beginning. This protects against malformed input. - // Any reasonable value here is fine. - std::string str; - if (!input->ReadString(&str, length)) { - return false; - } - unknown_fields->AddLengthDelimited(number, std::move(str)); } else { - if (!input->ReadRaw( - unknown_fields->AddLengthDelimitedUninitialized(number, length) - .data(), - length)) { + if (!input->ReadString(unknown_fields->AddLengthDelimited(number), + length)) { return false; } } @@ -375,10 +362,8 @@ bool WireFormat::SkipMessageSetField(io::CodedInputStream* input, UnknownFieldSet* unknown_fields) { uint32_t length; if (!input->ReadVarint32(&length)) return false; - return input->ReadRaw( - unknown_fields->AddLengthDelimitedUninitialized(field_number, length) - .data(), - length); + return input->ReadString(unknown_fields->AddLengthDelimited(field_number), + length); } bool WireFormat::ParseAndMergeMessageSetField(uint32_t field_number, From c5ca2cf8fbf92e44efa8b239bd03c0baf5fe62bc Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Jul 2024 07:13:03 -0700 Subject: [PATCH 082/107] Internal PiperOrigin-RevId: 655943906 --- rust/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/BUILD b/rust/BUILD index 089adeb170..b8ba669f66 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -33,6 +33,7 @@ rust_library( ":use_upb_kernel": ["--cfg=upb_kernel"], "//conditions:default": ["--cfg=cpp_kernel"], }), + visibility = ["//visibility:public"], deps = select({ ":use_upb_kernel": [":protobuf_upb"], "//conditions:default": [":protobuf_cpp"], From f1df391ad0b186fa119908fcf8966422dd4b5aff Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Jul 2024 09:41:20 -0700 Subject: [PATCH 083/107] Remove tests that have been disabled for 7 years Also remove check for GCC that is no longer supported. PiperOrigin-RevId: 655985282 --- src/google/protobuf/io/coded_stream_unittest.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc index 32c6594efa..8c7b65c961 100644 --- a/src/google/protobuf/io/coded_stream_unittest.cc +++ b/src/google/protobuf/io/coded_stream_unittest.cc @@ -337,9 +337,6 @@ TEST_2D(CodedStreamTest, WriteVarint64, kVarintCases, kBlockSizes) { memcmp(buffer_, kVarintCases_case.bytes, kVarintCases_case.size)); } -// This test causes gcc 3.3.5 (and earlier?) to give the cryptic error: -// "sorry, unimplemented: `method_call_expr' not supported by dump_expr" -#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3) int32_t kSignExtendedVarintCases[] = {0, 1, -1, 1237894, -37895138}; @@ -381,8 +378,6 @@ TEST_2D(CodedStreamTest, WriteVarint32SignExtended, kSignExtendedVarintCases, EXPECT_EQ(output.ByteCount(), input.ByteCount()); } -#endif - // ------------------------------------------------------------------- // Varint failure test. From 461450834b6db4181f5122c1470604d834cc9987 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Jul 2024 10:45:06 -0700 Subject: [PATCH 084/107] Introduce a prelude and remove the inherent impls for serialize() on msg, msgmut and msgview. PiperOrigin-RevId: 656007820 --- conformance/conformance_rust.rs | 9 +++++---- rust/BUILD | 11 +++++++---- rust/prelude.rs | 16 ++++++++++++++++ rust/protobuf.rs | 2 ++ rust/shared.rs | 2 ++ rust/test/cpp/interop/main.rs | 2 ++ rust/test/shared/BUILD | 16 ++++++++++++++++ rust/test/shared/child_parent_test.rs | 1 + rust/test/shared/serialization_test.rs | 2 ++ rust/test/shared/utf8/BUILD | 8 ++++---- rust/test/shared/utf8/utf8_test.rs | 4 +++- src/google/protobuf/compiler/rust/message.cc | 17 +++-------------- 12 files changed, 63 insertions(+), 27 deletions(-) create mode 100644 rust/prelude.rs diff --git a/conformance/conformance_rust.rs b/conformance/conformance_rust.rs index bbb0710970..1d428dfb32 100644 --- a/conformance/conformance_rust.rs +++ b/conformance/conformance_rust.rs @@ -7,13 +7,14 @@ use conformance_rust_proto::{ConformanceRequest, ConformanceResponse, WireFormat}; #[cfg(cpp_kernel)] -use protobuf_cpp as kernel; +use protobuf_cpp as protobuf; #[cfg(upb_kernel)] -use protobuf_upb as kernel; +use protobuf_upb as protobuf; -use kernel::Optional::{Set, Unset}; -use kernel::{Message, ParseError}; +use protobuf::prelude::*; +use protobuf::Optional::{Set, Unset}; +use protobuf::ParseError; use std::io::{self, ErrorKind, Read, Write}; use test_messages_edition2023_rust_proto::TestAllTypesEdition2023; diff --git a/rust/BUILD b/rust/BUILD index b8ba669f66..73acbf31ac 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -48,18 +48,21 @@ rust_library( # # shared.rs is the root of the crate and has public items re-exported in protobuf.rs for user use. PROTOBUF_SHARED = [ + # go/keep-sorted start "codegen_traits.rs", + "cord.rs", "enum.rs", "internal.rs", - "primitive.rs", + "map.rs", "optional.rs", - "cord.rs", + "prelude.rs", + "primitive.rs", + "proto_macro.rs", "proxied.rs", "repeated.rs", "shared.rs", "string.rs", - "map.rs", - "proto_macro.rs", + # go/keep-sorted end ] # The Rust Protobuf runtime using the upb kernel. diff --git a/rust/prelude.rs b/rust/prelude.rs new file mode 100644 index 0000000000..d32f2ae298 --- /dev/null +++ b/rust/prelude.rs @@ -0,0 +1,16 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +//! Prelude for the Protobuf Rust API. +//! This module contains only non-struct items that are needed for extremely +//! common usages of the generated types from application code. Any struct +//! or less common items should be imported normally. + +pub use crate::{ + proto, AsMut, AsView, Clear, ClearAndParse, IntoMut, IntoView, Message, MessageMut, + MessageView, Parse, Serialize, +}; diff --git a/rust/protobuf.rs b/rust/protobuf.rs index c83b94953a..13d0b5f14d 100644 --- a/rust/protobuf.rs +++ b/rust/protobuf.rs @@ -21,3 +21,5 @@ use protobuf_cpp as kernel; use protobuf_upb as kernel; pub use kernel::__public::*; + +pub use kernel::prelude; diff --git a/rust/shared.rs b/rust/shared.rs index db8655cd6b..4df25a9bff 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -45,6 +45,8 @@ pub mod __public { } pub use __public::*; +pub mod prelude; + /// Everything in `__internal` is allowed to change without it being considered /// a breaking change for the protobuf library. Nothing in here should be /// exported in `protobuf.rs`. diff --git a/rust/test/cpp/interop/main.rs b/rust/test/cpp/interop/main.rs index 93ec9e9f0e..7c9a4dda55 100644 --- a/rust/test/cpp/interop/main.rs +++ b/rust/test/cpp/interop/main.rs @@ -6,6 +6,8 @@ // https://developers.google.com/open-source/licenses/bsd use googletest::prelude::*; +use protobuf_cpp::prelude::*; + use protobuf_cpp::__runtime::{PtrAndLen, RawMessage}; use unittest_rust_proto::{TestAllExtensions, TestAllTypes, TestAllTypesMut, TestAllTypesView}; diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index 01b5610191..ecda92d474 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -45,7 +45,11 @@ rust_test( rust_test( name = "child_parent_upb_test", srcs = ["child_parent_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, deps = [ + "//rust:protobuf_upb_export", "//rust/test:child_upb_rust_proto", "//rust/test:parent_upb_rust_proto", "@crate_index//:googletest", @@ -55,7 +59,11 @@ rust_test( rust_test( name = "child_parent_cpp_test", srcs = ["child_parent_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, deps = [ + "//rust:protobuf_cpp_export", "//rust/test:child_cpp_rust_proto", "//rust/test:parent_cpp_rust_proto", "@crate_index//:googletest", @@ -263,10 +271,14 @@ rust_test( rust_test( name = "serialization_upb_test", srcs = ["serialization_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_upb_export", "//rust/test:unittest_edition_upb_rust_proto", "//rust/test:unittest_proto3_optional_upb_rust_proto", "//rust/test:unittest_proto3_upb_rust_proto", @@ -278,10 +290,14 @@ rust_test( rust_test( name = "serialization_cpp_test", srcs = ["serialization_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_cpp_export", "//rust/test:unittest_cpp_rust_proto", "//rust/test:unittest_edition_cpp_rust_proto", "//rust/test:unittest_proto3_cpp_rust_proto", diff --git a/rust/test/shared/child_parent_test.rs b/rust/test/shared/child_parent_test.rs index 82763b5c5c..b1905a9cb4 100644 --- a/rust/test/shared/child_parent_test.rs +++ b/rust/test/shared/child_parent_test.rs @@ -6,6 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd use googletest::prelude::*; +use protobuf::prelude::*; #[googletest::test] fn test_canonical_types() { diff --git a/rust/test/shared/serialization_test.rs b/rust/test/shared/serialization_test.rs index bf3d67a0b5..b0e882271a 100644 --- a/rust/test/shared/serialization_test.rs +++ b/rust/test/shared/serialization_test.rs @@ -6,6 +6,8 @@ // https://developers.google.com/open-source/licenses/bsd use googletest::prelude::*; +use protobuf::prelude::*; + use paste::paste; use unittest_edition_rust_proto::TestAllTypes as TestAllTypesEditions; use unittest_proto3_optional_rust_proto::TestProto3Optional; diff --git a/rust/test/shared/utf8/BUILD b/rust/test/shared/utf8/BUILD index 8d09729017..51f3b4da01 100644 --- a/rust/test/shared/utf8/BUILD +++ b/rust/test/shared/utf8/BUILD @@ -8,13 +8,13 @@ rust_test( name = "utf8_cpp_test", srcs = ["utf8_test.rs"], aliases = { - "//rust:protobuf_cpp": "protobuf", + "//rust:protobuf_cpp_export": "protobuf", }, deps = [ ":feature_verify_cpp_rust_proto", ":no_features_proto2_cpp_rust_proto", ":no_features_proto3_cpp_rust_proto", - "//rust:protobuf_cpp", + "//rust:protobuf_cpp_export", "@crate_index//:googletest", ], ) @@ -23,13 +23,13 @@ rust_test( name = "utf8_upb_test", srcs = ["utf8_test.rs"], aliases = { - "//rust:protobuf_upb": "protobuf", + "//rust:protobuf_upb_export": "protobuf", }, deps = [ ":feature_verify_upb_rust_proto", ":no_features_proto2_upb_rust_proto", ":no_features_proto3_upb_rust_proto", - "//rust:protobuf_upb", + "//rust:protobuf_upb_export", "@crate_index//:googletest", ], ) diff --git a/rust/test/shared/utf8/utf8_test.rs b/rust/test/shared/utf8/utf8_test.rs index e4431efbe3..a213136cb1 100644 --- a/rust/test/shared/utf8/utf8_test.rs +++ b/rust/test/shared/utf8/utf8_test.rs @@ -13,8 +13,10 @@ // behavior. Do not assume that the Protobuf team is intentional about these // behaviors while b/304774814 is open. -use feature_verify_rust_proto::Verify; use googletest::prelude::*; +use protobuf::prelude::*; + +use feature_verify_rust_proto::Verify; use no_features_proto2_rust_proto::NoFeaturesProto2; use no_features_proto3_rust_proto::NoFeaturesProto3; use protobuf::{ParseError, ProtoStr}; diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 94f67a7fc2..fda9f6266d 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -973,7 +973,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { impl $pb$::Serialize for $Msg$ { fn serialize(&self) -> Result, $pb$::SerializeError> { - self.serialize() + $pb$::AsView::as_view(self).serialize() } } @@ -1020,7 +1020,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { impl $pb$::Serialize for $Msg$View<'_> { fn serialize(&self) -> Result, $pb$::SerializeError> { - self.serialize() + $Msg::serialize$ } } @@ -1035,10 +1035,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.msg } - pub fn serialize(&self) -> Result, $pb$::SerializeError> { - $Msg::serialize$ - } - pub fn to_owned(&self) -> $Msg$ { $pb$::IntoProxied::into_proxied(*self, $pbi$::Private) } @@ -1096,7 +1092,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { impl $pb$::Serialize for $Msg$Mut<'_> { fn serialize(&self) -> Result, $pb$::SerializeError> { - self.serialize() + $pb$::AsView::as_view(self).serialize() } } @@ -1128,10 +1124,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.inner } - pub fn serialize(&self) -> Result, $pb$::SerializeError> { - $pb$::AsView::as_view(self).serialize() - } - pub fn to_owned(&self) -> $Msg$ { $pb$::AsView::as_view(self).to_owned() } @@ -1200,9 +1192,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $raw_arena_getter_for_message$ - pub fn serialize(&self) -> Result, $pb$::SerializeError> { - self.as_view().serialize() - } #[deprecated = "Prefer Msg::parse(), or use the new name 'clear_and_parse' to parse into a pre-existing message."] pub fn deserialize(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { self.clear_and_parse(data) From a98c7f79aa8febefca9456b708e92b9f79f6607f Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Jul 2024 11:13:13 -0700 Subject: [PATCH 085/107] Internal PiperOrigin-RevId: 656018120 --- rust/defs.bzl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/rust/defs.bzl b/rust/defs.bzl index 23d841a187..15fa0be6d5 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -15,12 +15,6 @@ load( "rust_upb_proto_library_aspect", ) -visibility([ - "//experimental/...", - "//src/google/protobuf/...", - "//rust/...", -]) - def rust_proto_library(name, deps, **args): """Declares all the boilerplate needed to use Rust protobufs conveniently. From 1e6abadc2b1612b6f00fe861dae39601252765d8 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Thu, 25 Jul 2024 15:41:59 -0700 Subject: [PATCH 086/107] Automated rollback of commit 1f717248b761e4f37c7d9cb9773c83b9948d9ae9. PiperOrigin-RevId: 656120088 --- conformance/failure_list_objc.txt | 5 + objectivec/GPBMessage.m | 459 +++--------------- objectivec/GPBUnknownFieldSet.m | 14 +- .../GPBUnknownFieldSet_PackagePrivate.h | 4 +- 4 files changed, 77 insertions(+), 405 deletions(-) diff --git a/conformance/failure_list_objc.txt b/conformance/failure_list_objc.txt index e34501ead7..0dd4279587 100644 --- a/conformance/failure_list_objc.txt +++ b/conformance/failure_list_objc.txt @@ -1,2 +1,7 @@ # JSON input or output tests are skipped (in conformance_objc.m) as mobile # platforms don't support JSON wire format to avoid code bloat. + +Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 652c791ef0..b0367699dc 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -5,9 +5,9 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#import "GPBMessage.h" - #import +#import "GPBMessage_PackagePrivate.h" + #import #import #import @@ -20,18 +20,11 @@ #import "GPBDictionary_PackagePrivate.h" #import "GPBExtensionInternals.h" #import "GPBExtensionRegistry.h" -#import "GPBMessage_PackagePrivate.h" #import "GPBRootObject_PackagePrivate.h" -#import "GPBUnknownField.h" -#import "GPBUnknownFieldSet.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownFields_PackagePrivate.h" #import "GPBUtilities_PackagePrivate.h" -// TODO: Consider using on other functions to reduce bloat when -// some compiler optimizations are enabled. -#define GPB_NOINLINE __attribute__((noinline)) - // Returns a new instance that was automatically created by |autocreator| for // its field |field|. static GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, GPBMessage *autocreator, @@ -71,16 +64,7 @@ NSString *const GPBMessageExceptionMessageTooLarge = @interface GPBMessage () { @package - // Only one of these two is ever set, GPBUnknownFieldSet is being deprecated and will - // eventually be removed, but because that api support mutation, once the property is - // fetch it must continue to be used so any mutations will be honored in future operations - // on the message. - // Read only operations that access these two/cause things to migration between them should - // be protected with an @synchronized(self) block (that way the code also doesn't have to - // worry about throws). - NSMutableData *unknownFieldData_; GPBUnknownFieldSet *unknownFields_; - NSMutableDictionary *extensionMap_; // Readonly access to autocreatedExtensionMap_ is protected via readOnlyLock_. NSMutableDictionary *autocreatedExtensionMap_; @@ -119,6 +103,7 @@ static id GetOrCreateMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *fiel static id GetMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *field); static NSMutableDictionary *CloneExtensionMap(NSDictionary *extensionMap, NSZone *zone) __attribute__((ns_returns_retained)); +static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self); #if defined(DEBUG) && DEBUG static NSError *MessageError(NSInteger code, NSDictionary *userInfo) { @@ -148,178 +133,6 @@ static NSError *ErrorFromException(NSException *exception) { return error; } -// Helper to encode varints onto the mutable data, the max size need is 10 bytes. -GPB_NOINLINE -static uint8_t *EncodeVarintU64(uint64_t val, uint8_t *ptr) { - do { - uint8_t byte = val & 0x7fU; - val >>= 7; - if (val) byte |= 0x80U; - *(ptr++) = byte; - } while (val); - return ptr; -} - -// Helper to encode varints onto the mutable data, the max size need is 5 bytes. -GPB_NOINLINE -static uint8_t *EncodeVarintU32(uint32_t val, uint8_t *ptr) { - do { - uint8_t byte = val & 0x7fU; - val >>= 7; - if (val) byte |= 0x80U; - *(ptr++) = byte; - } while (val); - return ptr; -} - -// Helper to encode signed int32 values as varints onto the mutable data, the max size need is 10 -// bytes. -GPB_NOINLINE -static uint8_t *EncodeVarintS32(int32_t val, uint8_t *ptr) { - if (val >= 0) { - return EncodeVarintU32((uint32_t)val, ptr); - } else { - // Must sign-extend - int64_t extended = val; - return EncodeVarintU64((uint64_t)extended, ptr); - } -} - -GPB_NOINLINE -static void AddUnknownFieldVarint32(GPBMessage *self, uint32_t fieldNumber, int32_t value) { - if (self->unknownFields_) { - [self->unknownFields_ mergeVarintField:fieldNumber value:value]; - return; - } - uint8_t buf[20]; - uint8_t *ptr = buf; - ptr = EncodeVarintU32(GPBWireFormatMakeTag(fieldNumber, GPBWireFormatVarint), ptr); - ptr = EncodeVarintS32(value, ptr); - - if (self->unknownFieldData_ == nil) { - self->unknownFieldData_ = [[NSMutableData alloc] initWithCapacity:ptr - buf]; - GPBBecomeVisibleToAutocreator(self); - } - [self->unknownFieldData_ appendBytes:buf length:ptr - buf]; -} - -GPB_NOINLINE -static void AddUnknownFieldLengthDelimited(GPBMessage *self, uint32_t fieldNumber, NSData *value) { - if (self->unknownFields_) { - [self->unknownFields_ mergeLengthDelimited:fieldNumber value:value]; - return; - } - uint8_t buf[20]; - uint8_t *ptr = buf; - ptr = EncodeVarintU32(GPBWireFormatMakeTag(fieldNumber, GPBWireFormatLengthDelimited), ptr); - ptr = EncodeVarintU64((uint64_t)value.length, ptr); - - if (self->unknownFieldData_ == nil) { - self->unknownFieldData_ = [[NSMutableData alloc] initWithCapacity:(ptr - buf) + value.length]; - GPBBecomeVisibleToAutocreator(self); - } - [self->unknownFieldData_ appendBytes:buf length:ptr - buf]; - [self->unknownFieldData_ appendData:value]; -} - -GPB_NOINLINE -static void AddUnknownMessageSetEntry(GPBMessage *self, uint32_t typeId, NSData *value) { - if (self->unknownFields_) { - // Legacy Set does this odd storage for MessageSet. - [self->unknownFields_ mergeLengthDelimited:typeId value:value]; - return; - } - - uint8_t buf[60]; - uint8_t *ptr = buf; - ptr = EncodeVarintU32(GPBWireFormatMessageSetItemTag, ptr); - ptr = EncodeVarintU32(GPBWireFormatMessageSetTypeIdTag, ptr); - ptr = EncodeVarintU32(typeId, ptr); - ptr = EncodeVarintU32(GPBWireFormatMessageSetMessageTag, ptr); - ptr = EncodeVarintU64((uint64_t)value.length, ptr); - uint8_t *split = ptr; - - ptr = EncodeVarintU32(GPBWireFormatMessageSetItemEndTag, ptr); - uint8_t *end = ptr; - - if (self->unknownFieldData_ == nil) { - self->unknownFieldData_ = [[NSMutableData alloc] initWithCapacity:(end - buf) + value.length]; - GPBBecomeVisibleToAutocreator(self); - } - [self->unknownFieldData_ appendBytes:buf length:split - buf]; - [self->unknownFieldData_ appendData:value]; - [self->unknownFieldData_ appendBytes:split length:end - split]; -} - -GPB_NOINLINE -static void ParseUnknownField(GPBMessage *self, uint32_t tag, GPBCodedInputStream *input) { - if (self->unknownFields_) { - if (![self->unknownFields_ mergeFieldFrom:tag input:input]) { - GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unexpected end-group tag"); - } - return; - } - - uint8_t buf[20]; - uint8_t *ptr = buf; - ptr = EncodeVarintU32(tag, ptr); // All will need the tag - NSData *bytesToAppend = nil; - - GPBCodedInputStreamState *state = &input->state_; - - switch (GPBWireFormatGetTagWireType(tag)) { - case GPBWireFormatVarint: { - ptr = EncodeVarintU64(GPBCodedInputStreamReadUInt64(state), ptr); - break; - } - case GPBWireFormatFixed64: { - uint64_t value = GPBCodedInputStreamReadFixed64(state); - *(ptr++) = (uint8_t)(value) & 0xFF; - *(ptr++) = (uint8_t)(value >> 8) & 0xFF; - *(ptr++) = (uint8_t)(value >> 16) & 0xFF; - *(ptr++) = (uint8_t)(value >> 24) & 0xFF; - *(ptr++) = (uint8_t)(value >> 32) & 0xFF; - *(ptr++) = (uint8_t)(value >> 40) & 0xFF; - *(ptr++) = (uint8_t)(value >> 48) & 0xFF; - *(ptr++) = (uint8_t)(value >> 56) & 0xFF; - break; - } - case GPBWireFormatLengthDelimited: { - bytesToAppend = GPBCodedInputStreamReadRetainedBytes(state); - ptr = EncodeVarintU64((uint64_t)bytesToAppend.length, ptr); - break; - } - case GPBWireFormatStartGroup: { - bytesToAppend = GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy( - state, GPBWireFormatGetTagFieldNumber(tag)); - break; - } - case GPBWireFormatEndGroup: - GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unexpected end-group tag"); - break; - case GPBWireFormatFixed32: { - uint32_t value = GPBCodedInputStreamReadFixed32(state); - *(ptr++) = (uint8_t)(value) & 0xFF; - *(ptr++) = (uint8_t)(value >> 8) & 0xFF; - *(ptr++) = (uint8_t)(value >> 16) & 0xFF; - *(ptr++) = (uint8_t)(value >> 24) & 0xFF; - break; - } - } - - if (self->unknownFieldData_ == nil) { - self->unknownFieldData_ = - [[NSMutableData alloc] initWithCapacity:(ptr - buf) + bytesToAppend.length]; - GPBBecomeVisibleToAutocreator(self); - } - - [self->unknownFieldData_ appendBytes:buf length:ptr - buf]; - if (bytesToAppend) { - [self->unknownFieldData_ appendData:bytesToAppend]; - [bytesToAppend release]; - } -} - static void CheckExtension(GPBMessage *self, GPBExtensionDescriptor *extension) { if (![self isKindOfClass:extension.containingMessageClass]) { [NSException raise:NSInvalidArgumentException @@ -916,7 +729,8 @@ static void DecodeSingleValueFromInputStream(GPBExtensionDescriptor *extension, if (!enumDescriptor.isClosed || enumDescriptor.enumVerifier(val)) { nsValue = [[NSNumber alloc] initWithInt:val]; } else { - AddUnknownFieldVarint32(messageToGetExtension, extension->description_->fieldNumber, val); + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(messageToGetExtension); + [unknownFields mergeVarintField:extension->description_->fieldNumber value:val]; nsValue = nil; } break; @@ -1106,50 +920,12 @@ void GPBClearMessageAutocreator(GPBMessage *self) { self->autocreatorExtension_ = nil; } -GPB_NOINLINE -static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, - GPBUnknownFieldSet *targetSet) { - GPBUnknownFieldSet *unknownFields = targetSet ? targetSet : self->unknownFields_; - -#if defined(DEBUG) && DEBUG - NSCAssert(unknownFields != nil, @"Internal error: unknown fields not initialized."); -#endif - - BOOL isMessageSet = self.descriptor.isWireFormat; - GPBUnknownFieldSet *decodeInto = isMessageSet ? [[GPBUnknownFieldSet alloc] init] : unknownFields; - - GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; - @try { - [decodeInto mergeFromCodedInputStream:input]; - } @catch (NSException *exception) { -#if defined(DEBUG) && DEBUG - NSLog(@"%@: Internal exception while parsing the unknown fields into a Set: %@", [self class], - exception); -#endif - } - [input release]; - - if (isMessageSet) { - // Need to transform the groups back into how Message feeds the data into a MessageSet when - // doing a full MessageSet based decode. - GPBUnknownField *groupField = [decodeInto getField:GPBWireFormatMessageSetItem]; - for (GPBUnknownFieldSet *group in groupField.groupList) { - GPBUnknownField *typeIdField = [group getField:GPBWireFormatMessageSetTypeId]; - GPBUnknownField *messageField = [group getField:GPBWireFormatMessageSetMessage]; - if (typeIdField.varintList.count != 1 || messageField.lengthDelimitedList.count != 1) { -#if defined(DEBUG) && DEBUG - NSCAssert(NO, @"Internal error: MessageSet group missing typeId or message."); -#endif - continue; - } - int32_t fieldNumber = (int32_t)[typeIdField.varintList valueAtIndex:0]; - GPBUnknownField *messageSetField = [[GPBUnknownField alloc] initWithNumber:fieldNumber]; - [messageSetField addLengthDelimited:messageField.lengthDelimitedList[0]]; - [unknownFields addField:messageSetField]; - [messageSetField release]; - } - [decodeInto release]; +static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { + if (!self->unknownFields_) { + self->unknownFields_ = [[GPBUnknownFieldSet alloc] init]; + GPBBecomeVisibleToAutocreator(self); } + return self->unknownFields_; } @implementation GPBMessage @@ -1386,12 +1162,8 @@ static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, GPBMessage *result = [[descriptor.messageClass allocWithZone:zone] init]; [self copyFieldsInto:result zone:zone descriptor:descriptor]; - - @synchronized(self) { - result->unknownFields_ = [unknownFields_ copyWithZone:zone]; - result->unknownFieldData_ = [unknownFieldData_ mutableCopyWithZone:zone]; - } - + // Make immutable copies of the extra bits. + result->unknownFields_ = [unknownFields_ copyWithZone:zone]; result->extensionMap_ = CloneExtensionMap(extensionMap_, zone); return result; } @@ -1465,8 +1237,6 @@ static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, [extensionMap_ release]; extensionMap_ = nil; - [unknownFieldData_ release]; - unknownFieldData_ = nil; [unknownFields_ release]; unknownFields_ = nil; @@ -1670,20 +1440,11 @@ static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, sortedExtensions:sortedExtensions]; } } - @synchronized(self) { - if (unknownFieldData_) { -#if defined(DEBUG) && DEBUG - NSAssert(unknownFields_ == nil, @"Internal error both unknown states were set"); -#endif - [output writeRawData:unknownFieldData_]; - } else { - if (descriptor.isWireFormat) { - [unknownFields_ writeAsMessageSetTo:output]; - } else { - [unknownFields_ writeToCodedOutputStream:output]; - } - } - } // @synchronized(self) + if (descriptor.isWireFormat) { + [unknownFields_ writeAsMessageSetTo:output]; + } else { + [unknownFields_ writeToCodedOutputStream:output]; + } } - (void)writeDelimitedToOutputStream:(NSOutputStream *)output { @@ -2404,25 +2165,11 @@ static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, #pragma mark - Unknown Field Support - (GPBUnknownFieldSet *)unknownFields { - @synchronized(self) { - if (unknownFieldData_) { -#if defined(DEBUG) && DEBUG - NSAssert(unknownFields_ == nil, @"Internal error both unknown states were set"); -#endif - unknownFields_ = [[GPBUnknownFieldSet alloc] init]; - MergeUnknownFieldDataIntoFieldSet(self, unknownFieldData_, nil); - [unknownFieldData_ release]; - unknownFieldData_ = nil; - } - return unknownFields_; - } // @synchronized(self) + return unknownFields_; } - (void)setUnknownFields:(GPBUnknownFieldSet *)unknownFields { - if (unknownFields != unknownFields_ || unknownFieldData_ != nil) { - // Changing sets or clearing. - [unknownFieldData_ release]; - unknownFieldData_ = nil; + if (unknownFields != unknownFields_) { [unknownFields_ release]; unknownFields_ = [unknownFields copy]; GPBBecomeVisibleToAutocreator(self); @@ -2505,18 +2252,19 @@ static void MergeUnknownFieldDataIntoFieldSet(GPBMessage *self, NSData *data, } else { // The extension isn't in the registry, but it was well formed, so the whole group structure // get preserved as an unknown field. - + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); // rawBytes was created via a NoCopy, so it can be reusing a // subrange of another NSData that might go out of scope as things // unwind, so a copy is needed to ensure what is saved in the // unknown fields stays valid. NSData *cloned = [NSData dataWithData:rawBytes]; - AddUnknownMessageSetEntry(self, typeId, cloned); + [unknownFields mergeMessageSetMessage:typeId data:cloned]; } } - (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data { - AddUnknownFieldLengthDelimited(self, fieldNum, data); + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); + [unknownFields addUnknownMapEntry:fieldNum value:data]; } #pragma mark - MergeFromCodedInputStream Support @@ -2589,7 +2337,8 @@ static void MergeSingleFieldFromCodedInputStream(GPBMessage *self, GPBFieldDescr if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { GPBSetInt32IvarWithFieldPrivate(self, field, val); } else { - AddUnknownFieldVarint32(self, GPBFieldNumber(field), val); + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); + [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; } } } // switch @@ -2638,7 +2387,8 @@ static void MergeRepeatedPackedFieldFromCodedInputStream(GPBMessage *self, if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { [(GPBEnumArray *)genericArray addRawValue:val]; } else { - AddUnknownFieldVarint32(self, GPBFieldNumber(field), val); + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); + [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; } break; } @@ -2706,7 +2456,8 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( if (!GPBFieldIsClosedEnum(field) || [field isValidEnumValue:val]) { [(GPBEnumArray *)genericArray addRawValue:val]; } else { - AddUnknownFieldVarint32(self, GPBFieldNumber(field), val); + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); + [unknownFields mergeVarintField:GPBFieldNumber(field) value:val]; } break; } @@ -2831,7 +2582,11 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } } - ParseUnknownField(self, tag, input); + GPBUnknownFieldSet *unknownFields = GetOrMakeUnknownFields(self); + if (![unknownFields mergeFieldFrom:tag input:input]) { + [NSException raise:NSInternalInconsistencyException + format:@"Internal Error: Unable to parse unknown field %u", tag]; + } } // while(YES) } @@ -2953,29 +2708,10 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } // for(fields) // Unknown fields. - if (unknownFields_) { -#if defined(DEBUG) && DEBUG - NSAssert(unknownFieldData_ == nil, @"Internal error both unknown states were set"); -#endif - @synchronized(other) { - if (other->unknownFields_) { -#if defined(DEBUG) && DEBUG - NSAssert(other->unknownFieldData_ == nil, @"Internal error both unknown states were set"); -#endif - [unknownFields_ mergeUnknownFields:other->unknownFields_]; - } else if (other->unknownFieldData_) { - MergeUnknownFieldDataIntoFieldSet(self, other->unknownFieldData_, nil); - } - } // @synchronized(other) + if (!unknownFields_) { + [self setUnknownFields:other.unknownFields]; } else { - NSData *otherData = GPBMessageUnknownFieldsData(other); - if (otherData) { - if (unknownFieldData_) { - [unknownFieldData_ appendData:otherData]; - } else { - unknownFieldData_ = [otherData mutableCopy]; - } - } + [unknownFields_ mergeUnknownFields:other.unknownFields]; } // Extensions @@ -3147,72 +2883,15 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } } - // Mutation while another thread is doing read only access is invalid, so the only thing we - // need to guard against is concurrent r/o access, so we can grab the values (and retain them) - // so we have a version to compare against safely incase the second access causes the transform - // between internal states. - GPBUnknownFieldSet *selfUnknownFields; - NSData *selfUnknownFieldData; - @synchronized(self) { - selfUnknownFields = [unknownFields_ retain]; - selfUnknownFieldData = [unknownFieldData_ retain]; - } - GPBUnknownFieldSet *otherUnknownFields; - NSData *otherUnknownFieldData; - @synchronized(otherMsg) { - otherUnknownFields = [otherMsg->unknownFields_ retain]; - otherUnknownFieldData = [otherMsg->unknownFieldData_ retain]; - } -#if defined(DEBUG) && DEBUG && !defined(NS_BLOCK_ASSERTIONS) - if (selfUnknownFields) { - NSAssert(selfUnknownFieldData == nil, @"Internal error both unknown states were set"); - } - if (otherUnknownFields) { - NSAssert(otherUnknownFieldData == nil, @"Internal error both unknown states were set"); - } -#endif - // Since a developer can set the legacy unknownFieldSet, treat nil and empty as the same. - if (selfUnknownFields && selfUnknownFields.countOfFields == 0) { - [selfUnknownFields release]; - selfUnknownFields = nil; - } - if (otherUnknownFields && otherUnknownFields.countOfFields == 0) { - [otherUnknownFields release]; - otherUnknownFields = nil; - } - - BOOL result = YES; - - if (selfUnknownFieldData && otherUnknownFieldData) { - // Both had data, compare it. - result = [selfUnknownFieldData isEqual:otherUnknownFieldData]; - } else if (selfUnknownFields && otherUnknownFields) { - // Both had fields set, compare them. - result = [selfUnknownFields isEqual:otherUnknownFields]; - } else { - // At this point, we're done to one have a set/nothing, and the other having data/nothing. - GPBUnknownFieldSet *theSet = selfUnknownFields ? selfUnknownFields : otherUnknownFields; - NSData *theData = selfUnknownFieldData ? selfUnknownFieldData : otherUnknownFieldData; - if (theSet) { - if (theData) { - GPBUnknownFieldSet *tempSet = [[GPBUnknownFieldSet alloc] init]; - MergeUnknownFieldDataIntoFieldSet(self, theData, tempSet); - result = [tempSet isEqual:theSet]; - [tempSet release]; - } else { - result = NO; - } - } else { - // It was a data/nothing and nothing, so they equal if the other didn't have data. - result = theData == nil; + // nil and empty are equal + GPBUnknownFieldSet *otherUnknowns = otherMsg->unknownFields_; + if ([unknownFields_ countOfFields] != 0 || [otherUnknowns countOfFields] != 0) { + if (![unknownFields_ isEqual:otherUnknowns]) { + return NO; } } - [selfUnknownFields release]; - [selfUnknownFieldData release]; - [otherUnknownFields release]; - [otherUnknownFieldData release]; - return result; + return YES; } // It is very difficult to implement a generic hash for ProtoBuf messages that @@ -3475,20 +3154,11 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream( } // for(fields) // Add any unknown fields. - @synchronized(self) { - if (unknownFieldData_) { -#if defined(DEBUG) && DEBUG - NSAssert(unknownFields_ == nil, @"Internal error both unknown states were set"); -#endif - result += [unknownFieldData_ length]; - } else { - if (descriptor.wireFormat) { - result += [unknownFields_ serializedSizeAsMessageSet]; - } else { - result += [unknownFields_ serializedSize]; - } - } - } // @synchronized(self) + if (descriptor.wireFormat) { + result += [unknownFields_ serializedSizeAsMessageSet]; + } else { + result += [unknownFields_ serializedSize]; + } // Add any extensions. for (GPBExtensionDescriptor *extension in extensionMap_) { @@ -4016,29 +3686,20 @@ id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { NSData *GPBMessageUnknownFieldsData(GPBMessage *self) { NSData *result = nil; - @synchronized(self) { - GPBUnknownFieldSet *unknownFields = self->unknownFields_; - if (unknownFields) { -#if defined(DEBUG) && DEBUG - NSCAssert(self->unknownFieldData_ == nil, @"Internal error both unknown states were set"); -#endif - if (self.descriptor.isWireFormat) { - NSMutableData *mutableData = - [NSMutableData dataWithLength:unknownFields.serializedSizeAsMessageSet]; - GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; - [unknownFields writeAsMessageSetTo:output]; - [output flush]; - [output release]; - result = mutableData; - } else { - result = [unknownFields data]; - } + GPBUnknownFieldSet *unknownFields = self->unknownFields_; + if (unknownFields) { + if (self.descriptor.isWireFormat) { + NSMutableData *mutableData = + [NSMutableData dataWithLength:unknownFields.serializedSizeAsMessageSet]; + GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; + [unknownFields writeAsMessageSetTo:output]; + [output flush]; + [output release]; + result = mutableData; } else { - // Internally we can borrow it without a copy since this is immediately used by callers - // and multithreaded access with any mutation is not allow on messages. - result = self->unknownFieldData_; + result = [unknownFields data]; } - } // @synchronized(self) + } return result; } diff --git a/objectivec/GPBUnknownFieldSet.m b/objectivec/GPBUnknownFieldSet.m index 6ec70b8925..10fb80c0c7 100644 --- a/objectivec/GPBUnknownFieldSet.m +++ b/objectivec/GPBUnknownFieldSet.m @@ -271,11 +271,6 @@ static void GPBUnknownFieldSetMergeUnknownFields(__unused const void *key, const [[self mutableFieldForNumber:number create:YES] addVarint:value]; } -- (void)mergeLengthDelimited:(int32_t)fieldNum value:(NSData *)value { - checkNumber(fieldNum); - [[self mutableFieldForNumber:fieldNum create:YES] addLengthDelimited:value]; -} - - (BOOL)mergeFieldFrom:(int32_t)tag input:(GPBCodedInputStream *)input { NSAssert(GPBWireFormatIsValidTag(tag), @"Got passed an invalid tag"); int32_t number = GPBWireFormatGetTagFieldNumber(tag); @@ -318,6 +313,15 @@ static void GPBUnknownFieldSetMergeUnknownFields(__unused const void *key, const } } +- (void)mergeMessageSetMessage:(int32_t)number data:(NSData *)messageData { + [[self mutableFieldForNumber:number create:YES] addLengthDelimited:messageData]; +} + +- (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data { + GPBUnknownField *field = [self mutableFieldForNumber:fieldNum create:YES]; + [field addLengthDelimited:data]; +} + - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input { while (YES) { int32_t tag = GPBCodedInputStreamReadTag(&input->state_); diff --git a/objectivec/GPBUnknownFieldSet_PackagePrivate.h b/objectivec/GPBUnknownFieldSet_PackagePrivate.h index 1d5ff50f45..7de20ed0cf 100644 --- a/objectivec/GPBUnknownFieldSet_PackagePrivate.h +++ b/objectivec/GPBUnknownFieldSet_PackagePrivate.h @@ -27,7 +27,9 @@ - (void)mergeFromCodedInputStream:(GPBCodedInputStream *)input; - (void)mergeVarintField:(int32_t)number value:(int32_t)value; -- (void)mergeLengthDelimited:(int32_t)number value:(NSData *)value; - (BOOL)mergeFieldFrom:(int32_t)tag input:(GPBCodedInputStream *)input; +- (void)mergeMessageSetMessage:(int32_t)number data:(NSData *)messageData; + +- (void)addUnknownMapEntry:(int32_t)fieldNum value:(NSData *)data; @end From 6f58085577ad8191f502670c993cf0da72dec634 Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Fri, 26 Jul 2024 08:09:33 -0700 Subject: [PATCH 087/107] Adds an initial Cargo.toml for the protobuf crate. Adds a new 'bzl' feature that is used to adjust import paths that need to change in the Cargo build vs Blaze build. This protobuf rust crate is a single crate that merges all of our current crates (protobuf, rust/upb, and utf8). PiperOrigin-RevId: 656405153 --- rust/BUILD | 7 ++++++- rust/cargo/Cargo.toml | 20 ++++++++++++++++++++ rust/cargo/build.rs | 16 ++++++++++++++++ rust/proxied.rs | 6 +++--- rust/shared.rs | 16 ++++++++++++++-- rust/string.rs | 5 ++--- rust/upb.rs | 3 +++ rust/upb/BUILD | 1 + rust/upb/lib.rs | 1 + rust/utf8.rs | 4 ++-- 10 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 rust/cargo/Cargo.toml create mode 100644 rust/cargo/build.rs diff --git a/rust/BUILD b/rust/BUILD index 73acbf31ac..d403bb3d5c 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -77,7 +77,10 @@ rust_library( proc_macro_deps = [ "@crate_index//:paste", ], - rustc_flags = ["--cfg=upb_kernel"], + rustc_flags = [ + "--cfg=upb_kernel", + "--cfg=bzl", + ], visibility = [ "//rust:__subpackages__", "//src/google/protobuf:__subpackages__", @@ -93,6 +96,7 @@ rust_test( crate = ":protobuf_upb", rustc_flags = [ "--cfg=upb_kernel", + "--cfg=bzl", ], deps = [ "@crate_index//:googletest", @@ -136,6 +140,7 @@ rust_test( crate = ":protobuf_cpp", rustc_flags = [ "--cfg=cpp_kernel", + "--cfg=bzl", ], deps = [ "@crate_index//:googletest", diff --git a/rust/cargo/Cargo.toml b/rust/cargo/Cargo.toml new file mode 100644 index 0000000000..2d8e44bb9d --- /dev/null +++ b/rust/cargo/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "protobuf" +version = "4.27.3-beta.0" +edition = "2021" +links = "upb" + +[lib] +path = "src/shared.rs" + +[dependencies] +paste = "1.0.15" + +[dev-dependencies] +googletest = {git = "https://github.com/google/googletest-rust.git" } + +[build-dependencies] +cmake = "0.1.50" + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bzl)', 'cfg(cpp_kernel)', 'cfg(upb_kernel)'] } \ No newline at end of file diff --git a/rust/cargo/build.rs b/rust/cargo/build.rs new file mode 100644 index 0000000000..e400e15471 --- /dev/null +++ b/rust/cargo/build.rs @@ -0,0 +1,16 @@ +fn main() { + cc::Build::new() + .flag("-std=c99") + // TODO: Come up with a way to enable lto + // .flag("-flto=thin") + .warnings(false) + .include("src") + .file("src/upb.c") + .file("src/utf8_range.c") + .file("src/upb/upb_api.c") + .define("UPB_BUILD_API", Some("1")) + .define("PROTOBUF_CARGO", Some("1")) + .compile("upb"); + let path = std::path::Path::new("src"); + println!("cargo:include={}", path.canonicalize().unwrap().display()) +} diff --git a/rust/proxied.rs b/rust/proxied.rs index be941fa3cd..0267b5fc71 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -108,7 +108,7 @@ pub trait AsView { /// /// For example, the call to `.as_view()` in the following snippet /// wouldn't be necessary in concrete code: - /// ``` + /// ```ignore /// fn reborrow<'a, 'b, T>(x: &'b View<'a, T>) -> View<'b, T> /// where 'a: 'b, T: Proxied /// { @@ -136,7 +136,7 @@ pub trait IntoView<'msg>: AsView { /// `into_view` to explicitly perform the operation that in concrete /// code coercion would perform implicitly. /// - /// ``` + /// ```ignore /// fn reborrow_generic_view_into_view<'a, 'b, T>( /// x: View<'a, T>, /// y: View<'b, T>, @@ -182,7 +182,7 @@ pub trait IntoMut<'msg>: AsMut { /// `into_mut` to explicitly perform the operation that in concrete code /// coercion would perform implicitly. /// - /// ``` + /// ```ignore /// fn reborrow_generic_mut_into_mut<'a, 'b, T>(x: Mut<'a, T>, y: Mut<'b, T>) -> [Mut<'b, T>; 2] /// where /// T: Proxied, diff --git a/rust/shared.rs b/rust/shared.rs index 4df25a9bff..80dd3827dd 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -56,10 +56,10 @@ pub mod __internal; /// Everything in `__runtime` is allowed to change without it being considered /// a breaking change for the protobuf library. Nothing in here should be /// exported in `protobuf.rs`. -#[cfg(cpp_kernel)] +#[cfg(all(bzl, cpp_kernel))] #[path = "cpp.rs"] pub mod __runtime; -#[cfg(upb_kernel)] +#[cfg(any(not(bzl), upb_kernel))] #[path = "upb.rs"] pub mod __runtime; @@ -75,6 +75,18 @@ mod proxied; mod repeated; mod string; +#[cfg(not(bzl))] +#[path = "upb/lib.rs"] +mod upb; + +#[cfg(not(bzl))] +mod utf8; + +// Forces the utf8 crate to be accessible from crate::. +#[cfg(bzl)] +#[allow(clippy::single_component_path_imports)] +use utf8; + // If the Upb and C++ kernels are both linked into the same binary, this symbol // will be defined twice and cause a link error. #[no_mangle] diff --git a/rust/string.rs b/rust/string.rs index 748fe9b71e..b25e548392 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -12,8 +12,8 @@ use crate::__internal::Private; use crate::__runtime::{InnerProtoString, PtrAndLen, RawMessage}; use crate::{ - AsView, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Optional, Proxied, Proxy, View, - ViewProxy, + utf8::Utf8Chunks, AsView, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Optional, Proxied, + Proxy, View, ViewProxy, }; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; @@ -26,7 +26,6 @@ use std::ops::{Deref, DerefMut}; use std::ptr; use std::rc::Rc; use std::sync::Arc; -use utf8::Utf8Chunks; pub struct ProtoBytes { pub(crate) inner: InnerProtoString, diff --git a/rust/upb.rs b/rust/upb.rs index 0af32ba382..eda6fc4e27 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -19,7 +19,10 @@ use std::ptr::{self, NonNull}; use std::slice; use std::sync::OnceLock; +#[cfg(bzl)] extern crate upb; +#[cfg(not(bzl))] +use crate::upb; // Temporarily 'pub' since a lot of gencode is directly calling any of the ffi // fns. diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 8fa34f6237..3cd2c5e84b 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -28,6 +28,7 @@ rust_library( "text.rs", "wire.rs", ], + rustc_flags = ["--cfg=bzl"], visibility = [ "//rust:__subpackages__", "//src/google/protobuf:__subpackages__", diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 650baec854..67319afe6d 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -5,6 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd #![deny(unsafe_op_in_unsafe_fn)] +#![cfg_attr(not(bzl), allow(unused_imports))] mod arena; diff --git a/rust/utf8.rs b/rust/utf8.rs index a452e8f916..7a169f0941 100644 --- a/rust/utf8.rs +++ b/rust/utf8.rs @@ -30,7 +30,7 @@ use std::str::from_utf8_unchecked; /// /// # Examples /// -/// ``` +/// ```ignore /// use googletest::prelude::*; /// use utf8::Utf8Chunks; /// @@ -133,7 +133,7 @@ impl fmt::Debug for Debug<'_> { /// This can be used to create functionality similar to /// [`String::from_utf8_lossy`] without allocating heap memory: /// -/// ``` +/// ```ignore /// use utf8::Utf8Chunks; /// /// fn from_utf8_lossy(input: &[u8], mut push: F) where F: FnMut(&str) { From 9bd81567cf58c22b713c3af9f0854ede96614050 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 26 Jul 2024 09:44:40 -0700 Subject: [PATCH 088/107] Remove deprecated .deseralize() fn PiperOrigin-RevId: 656433799 --- src/google/protobuf/compiler/rust/message.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index fda9f6266d..a6dca4b460 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -1192,10 +1192,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $raw_arena_getter_for_message$ - #[deprecated = "Prefer Msg::parse(), or use the new name 'clear_and_parse' to parse into a pre-existing message."] - pub fn deserialize(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { - self.clear_and_parse(data) - } pub fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { $Msg::clear_and_parse$ } From 1aedb97602b084b51c903c6c82b89e39b6d5078a Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Fri, 26 Jul 2024 10:36:47 -0700 Subject: [PATCH 089/107] Move python WKT test proto messages into well_known_types_test.proto PiperOrigin-RevId: 656452263 --- .../google/protobuf/internal/any_test.proto | 28 ------- .../protobuf/internal/more_messages.proto | 7 -- .../protobuf/internal/text_format_test.py | 2 +- .../internal/well_known_types_test.proto | 29 +++++++ .../internal/well_known_types_test.py | 76 ++++++++++--------- 5 files changed, 69 insertions(+), 73 deletions(-) delete mode 100644 python/google/protobuf/internal/any_test.proto create mode 100644 python/google/protobuf/internal/well_known_types_test.proto diff --git a/python/google/protobuf/internal/any_test.proto b/python/google/protobuf/internal/any_test.proto deleted file mode 100644 index 6c21ef8721..0000000000 --- a/python/google/protobuf/internal/any_test.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -// Author: jieluo@google.com (Jie Luo) - -syntax = "proto2"; - -package google.protobuf.internal; - -import "google/protobuf/any.proto"; - -message TestAny { - optional google.protobuf.Any value = 1; - optional int32 int_value = 2; - map map_value = 3; - extensions 10 to max; -} - -message TestAnyExtension1 { - extend TestAny { - optional TestAnyExtension1 extension1 = 98418603; - } - optional int32 i = 15; -} diff --git a/python/google/protobuf/internal/more_messages.proto b/python/google/protobuf/internal/more_messages.proto index fec0d72495..6ce788c927 100644 --- a/python/google/protobuf/internal/more_messages.proto +++ b/python/google/protobuf/internal/more_messages.proto @@ -352,10 +352,3 @@ message ConflictJsonName { optional int32 value = 1 [json_name = "old_value"]; optional int32 new_value = 2 [json_name = "value"]; } - -message WKTMessage { - optional Timestamp optional_timestamp = 1; - optional Duration optional_duration = 2; - optional Struct optional_struct = 3; - optional ListValue optional_list_value = 4; -} diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index a527e5bd67..cb2efb3a3b 100644 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -19,7 +19,7 @@ import unittest from google.protobuf import any_pb2 from google.protobuf import struct_pb2 from google.protobuf import descriptor_pb2 -from google.protobuf.internal import any_test_pb2 as test_extend_any +from google.protobuf.internal import well_known_types_test_pb2 as test_extend_any from google.protobuf.internal import api_implementation from google.protobuf.internal import message_set_extensions_pb2 from google.protobuf.internal import test_proto3_optional_pb2 diff --git a/python/google/protobuf/internal/well_known_types_test.proto b/python/google/protobuf/internal/well_known_types_test.proto new file mode 100644 index 0000000000..dad4f53c72 --- /dev/null +++ b/python/google/protobuf/internal/well_known_types_test.proto @@ -0,0 +1,29 @@ +edition = "2023"; + +package google.protobuf.internal; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +message TestAny { + google.protobuf.Any value = 1; + int32 int_value = 2; + map map_value = 3; + extensions 10 to max; +} + +message TestAnyExtension1 { + extend TestAny { + TestAnyExtension1 extension1 = 98418603; + } + int32 i = 15; +} + +message WKTMessage { + Timestamp optional_timestamp = 1; + Duration optional_duration = 2; + Struct optional_struct = 3; + ListValue optional_list_value = 4; +} diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 971ef8d788..1f05245f0d 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -14,9 +14,9 @@ import datetime import unittest from google.protobuf import text_format -from google.protobuf.internal import any_test_pb2 from google.protobuf.internal import more_messages_pb2 from google.protobuf.internal import well_known_types +from google.protobuf.internal import well_known_types_test_pb2 from google.protobuf import any_pb2 from google.protobuf import duration_pb2 @@ -361,7 +361,7 @@ class TimeUtilTest(TimeUtilTestBase): ) def testTimestampAssignment(self, date_parts, tzinfo): original_datetime = datetime.datetime(*date_parts, tzinfo=tzinfo) # pylint:disable=g-tzinfo-datetime - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_timestamp = original_datetime self.assertEqual(7200, msg.optional_timestamp.seconds) self.assertEqual(0, msg.optional_timestamp.nanos) @@ -374,11 +374,13 @@ class TimeUtilTest(TimeUtilTestBase): ) def testTimestampCreation(self, date_parts, tzinfo): original_datetime = datetime.datetime(*date_parts, tzinfo=tzinfo) # pylint:disable=g-tzinfo-datetime - msg = more_messages_pb2.WKTMessage(optional_timestamp=original_datetime) + msg = well_known_types_test_pb2.WKTMessage( + optional_timestamp=original_datetime + ) self.assertEqual(7200, msg.optional_timestamp.seconds) self.assertEqual(0, msg.optional_timestamp.nanos) - msg2 = more_messages_pb2.WKTMessage( + msg2 = well_known_types_test_pb2.WKTMessage( optional_timestamp=msg.optional_timestamp ) self.assertEqual(7200, msg2.optional_timestamp.seconds) @@ -401,24 +403,24 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testTimestampAdd(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_timestamp = old_time # Timestamp + timedelta - new_msg1 = more_messages_pb2.WKTMessage() + new_msg1 = well_known_types_test_pb2.WKTMessage() new_msg1.optional_timestamp = msg.optional_timestamp + time_delta self.assertEqual(expected_sec, new_msg1.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg1.optional_timestamp.nanos) # timedelta + Timestamp - new_msg2 = more_messages_pb2.WKTMessage() + new_msg2 = well_known_types_test_pb2.WKTMessage() new_msg2.optional_timestamp = time_delta + msg.optional_timestamp self.assertEqual(expected_sec, new_msg2.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg2.optional_timestamp.nanos) # Timestamp + Duration msg.optional_duration.FromTimedelta(time_delta) - new_msg3 = more_messages_pb2.WKTMessage() + new_msg3 = well_known_types_test_pb2.WKTMessage() new_msg3.optional_timestamp = msg.optional_timestamp + msg.optional_duration self.assertEqual(expected_sec, new_msg3.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg3.optional_timestamp.nanos) @@ -440,23 +442,23 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testTimestampSub(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_timestamp = old_time # Timestamp - timedelta - new_msg1 = more_messages_pb2.WKTMessage() + new_msg1 = well_known_types_test_pb2.WKTMessage() new_msg1.optional_timestamp = msg.optional_timestamp - time_delta self.assertEqual(expected_sec, new_msg1.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg1.optional_timestamp.nanos) # Timestamp - Duration msg.optional_duration = time_delta - new_msg2 = more_messages_pb2.WKTMessage() + new_msg2 = well_known_types_test_pb2.WKTMessage() new_msg2.optional_timestamp = msg.optional_timestamp - msg.optional_duration self.assertEqual(expected_sec, new_msg2.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg2.optional_timestamp.nanos) - result_msg = more_messages_pb2.WKTMessage() + result_msg = well_known_types_test_pb2.WKTMessage() result_msg.optional_timestamp = old_time - time_delta # Timestamp - Timestamp td = msg.optional_timestamp - result_msg.optional_timestamp @@ -532,12 +534,12 @@ class TimeUtilTest(TimeUtilTestBase): message.ToJsonString) self.assertRaisesRegex(ValueError, 'Timestamp is not valid', message.FromSeconds, -62135596801) - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() with self.assertRaises(AttributeError): msg.optional_timestamp = 1 with self.assertRaises(AttributeError): - msg2 = more_messages_pb2.WKTMessage(optional_timestamp=1) + msg2 = well_known_types_test_pb2.WKTMessage(optional_timestamp=1) with self.assertRaises(TypeError): msg.optional_timestamp + '' @@ -577,12 +579,12 @@ class TimeUtilTest(TimeUtilTestBase): self.assertRaisesRegex(ValueError, r'Duration is not valid\: Sign mismatch.', message.ToJsonString) - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() with self.assertRaises(AttributeError): msg.optional_duration = 1 with self.assertRaises(AttributeError): - msg2 = more_messages_pb2.WKTMessage(optional_duration=1) + msg2 = well_known_types_test_pb2.WKTMessage(optional_duration=1) with self.assertRaises(TypeError): msg.optional_duration + '' @@ -594,7 +596,7 @@ class TimeUtilTest(TimeUtilTestBase): ('test1', -1999999, -1, -999999000), ('test2', 1999999, 1, 999999000) ) def testDurationAssignment(self, microseconds, expected_sec, expected_nano): - message = more_messages_pb2.WKTMessage() + message = well_known_types_test_pb2.WKTMessage() expected_td = datetime.timedelta(microseconds=microseconds) message.optional_duration = expected_td self.assertEqual(expected_td, message.optional_duration.ToTimedelta()) @@ -605,7 +607,7 @@ class TimeUtilTest(TimeUtilTestBase): ('test1', -1999999, -1, -999999000), ('test2', 1999999, 1, 999999000) ) def testDurationCreation(self, microseconds, expected_sec, expected_nano): - message = more_messages_pb2.WKTMessage( + message = well_known_types_test_pb2.WKTMessage( optional_duration=datetime.timedelta(microseconds=microseconds) ) expected_td = datetime.timedelta(microseconds=microseconds) @@ -630,24 +632,24 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testDurationAdd(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_duration = time_delta msg.optional_timestamp = old_time # Duration + datetime - msg1 = more_messages_pb2.WKTMessage() + msg1 = well_known_types_test_pb2.WKTMessage() msg1.optional_timestamp = msg.optional_duration + old_time self.assertEqual(expected_sec, msg1.optional_timestamp.seconds) self.assertEqual(expected_nano, msg1.optional_timestamp.nanos) # datetime + Duration - msg2 = more_messages_pb2.WKTMessage() + msg2 = well_known_types_test_pb2.WKTMessage() msg2.optional_timestamp = old_time + msg.optional_duration self.assertEqual(expected_sec, msg2.optional_timestamp.seconds) self.assertEqual(expected_nano, msg2.optional_timestamp.nanos) # Duration + Timestamp - msg3 = more_messages_pb2.WKTMessage() + msg3 = well_known_types_test_pb2.WKTMessage() msg3.optional_timestamp = msg.optional_duration + msg.optional_timestamp self.assertEqual(expected_sec, msg3.optional_timestamp.seconds) self.assertEqual(expected_nano, msg3.optional_timestamp.nanos) @@ -669,7 +671,7 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testDurationSub(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_duration = time_delta # datetime - Duration @@ -843,7 +845,7 @@ class StructTest(unittest.TestCase): 'key2': 'abc', 'key3': {'subkey': 11.0, 'k': False}, } - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_struct = dictionary self.assertEqual(msg.optional_struct, dictionary) @@ -855,7 +857,7 @@ class StructTest(unittest.TestCase): self.assertEqual(msg.optional_struct, dictionary2) # Tests assign empty - msg2 = more_messages_pb2.WKTMessage() + msg2 = well_known_types_test_pb2.WKTMessage() self.assertNotIn('optional_struct', msg2) msg2.optional_struct = {} self.assertIn('optional_struct', msg2) @@ -863,7 +865,7 @@ class StructTest(unittest.TestCase): def testListValueAssignment(self): list_value = [6, 'seven', True, False, None, {}] - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_list_value = list_value self.assertEqual(msg.optional_list_value, list_value) @@ -874,7 +876,7 @@ class StructTest(unittest.TestCase): 'key3': {'subkey': 11.0, 'k': False}, } list_value = [6, 'seven', True, False, None, dictionary] - msg = more_messages_pb2.WKTMessage( + msg = well_known_types_test_pb2.WKTMessage( optional_struct=dictionary, optional_list_value=list_value ) self.assertEqual(len(msg.optional_struct), len(dictionary)) @@ -882,7 +884,7 @@ class StructTest(unittest.TestCase): self.assertEqual(len(msg.optional_list_value), len(list_value)) self.assertEqual(msg.optional_list_value, list_value) - msg2 = more_messages_pb2.WKTMessage( + msg2 = well_known_types_test_pb2.WKTMessage( optional_struct={}, optional_list_value=[] ) self.assertIn('optional_struct', msg2) @@ -892,17 +894,17 @@ class StructTest(unittest.TestCase): def testSpecialStructConstruct(self): dictionary = {'key1': 6.0} - msg = more_messages_pb2.WKTMessage(optional_struct=dictionary) + msg = well_known_types_test_pb2.WKTMessage(optional_struct=dictionary) self.assertEqual(msg.optional_struct, dictionary) dictionary2 = {'fields': 7.0} - msg2 = more_messages_pb2.WKTMessage(optional_struct=dictionary2) + msg2 = well_known_types_test_pb2.WKTMessage(optional_struct=dictionary2) self.assertEqual(msg2.optional_struct, dictionary2) # Construct Struct as normal message value_msg = struct_pb2.Value(number_value=5.0) dictionary3 = {'fields': {'key1': value_msg}} - msg3 = more_messages_pb2.WKTMessage(optional_struct=dictionary3) + msg3 = well_known_types_test_pb2.WKTMessage(optional_struct=dictionary3) self.assertEqual(msg3.optional_struct, {'key1': 5.0}) def testMergeFrom(self): @@ -955,7 +957,7 @@ class AnyTest(unittest.TestCase): def testAnyMessage(self): # Creates and sets message. - msg = any_test_pb2.TestAny() + msg = well_known_types_test_pb2.TestAny() msg_descriptor = msg.DESCRIPTOR all_types = unittest_pb2.TestAllTypes() all_descriptor = all_types.DESCRIPTOR @@ -985,7 +987,7 @@ class AnyTest(unittest.TestCase): msg_descriptor.full_name) def testUnpackWithNoSlashInTypeUrl(self): - msg = any_test_pb2.TestAny() + msg = well_known_types_test_pb2.TestAny() all_types = unittest_pb2.TestAllTypes() all_descriptor = all_types.DESCRIPTOR msg.value.Pack(all_types) @@ -997,14 +999,14 @@ class AnyTest(unittest.TestCase): def testMessageName(self): # Creates and sets message. - submessage = any_test_pb2.TestAny() + submessage = well_known_types_test_pb2.TestAny() submessage.int_value = 12345 msg = any_pb2.Any() msg.Pack(submessage) self.assertEqual(msg.TypeName(), 'google.protobuf.internal.TestAny') def testPackWithCustomTypeUrl(self): - submessage = any_test_pb2.TestAny() + submessage = well_known_types_test_pb2.TestAny() submessage.int_value = 12345 msg = any_pb2.Any() # Pack with a custom type URL prefix. @@ -1020,12 +1022,12 @@ class AnyTest(unittest.TestCase): self.assertEqual(msg.type_url, '/%s' % submessage.DESCRIPTOR.full_name) # Test unpacking the type. - unpacked_message = any_test_pb2.TestAny() + unpacked_message = well_known_types_test_pb2.TestAny() self.assertTrue(msg.Unpack(unpacked_message)) self.assertEqual(submessage, unpacked_message) def testPackDeterministic(self): - submessage = any_test_pb2.TestAny() + submessage = well_known_types_test_pb2.TestAny() for i in range(10): submessage.map_value[str(i)] = i * 2 msg = any_pb2.Any() From 0fbba32b3e6dfb5da4077f428590811dd17fa7a6 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 26 Jul 2024 10:39:35 -0700 Subject: [PATCH 090/107] Add Message::Clear() and MessageMut::Clear() (behind the Clear trait) Distinct from any clear_submsg(), this clears the message contents and doesn't affect presence on parent (and also allows for clearing owned messages which don't exist as a field to be cleared). PiperOrigin-RevId: 656453234 --- rust/codegen_traits.rs | 11 +++--- rust/test/shared/accessors_proto3_test.rs | 33 +++++++++++++++- rust/test/shared/accessors_test.rs | 28 +++++++++++++ rust/upb/lib.rs | 5 ++- rust/upb/message.rs | 28 ++++++++++++- src/google/protobuf/compiler/rust/message.cc | 41 ++++++++++++++++++++ 6 files changed, 136 insertions(+), 10 deletions(-) diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index ddfe3cfd17..ad8dd64a87 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -11,7 +11,7 @@ use crate::{MutProxied, MutProxy, ViewProxy}; use create::Parse; use read::Serialize; use std::fmt::Debug; -use write::ClearAndParse; +use write::{Clear, ClearAndParse}; /// A trait that all generated owned message types implement. pub trait Message: MutProxied @@ -20,8 +20,7 @@ pub trait Message: MutProxied // Read traits: + Debug + Serialize // Write traits: - // TODO: Msg should impl Clear. - + ClearAndParse + + Clear + ClearAndParse // Thread safety: + Send + Sync // Copy/Clone: @@ -47,10 +46,12 @@ pub trait MessageMut<'msg>: // Read traits: + Debug + Serialize // Write traits: - // TODO: MsgMut should impl Clear and ClearAndParse. + // TODO: MsgMut should impl ClearAndParse. + + Clear // Thread safety: + Sync - // Copy/Clone: (Neither) + // Copy/Clone: + // (Neither) { #[doc(hidden)] type Message: Message; diff --git a/rust/test/shared/accessors_proto3_test.rs b/rust/test/shared/accessors_proto3_test.rs index 63aaf1acf5..1df120d1a1 100644 --- a/rust/test/shared/accessors_proto3_test.rs +++ b/rust/test/shared/accessors_proto3_test.rs @@ -5,9 +5,12 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -/// Tests covering accessors for singular bool, int32, int64, and bytes fields -/// on proto3. +//! Tests covering accessors for singular bool, int32, int64, and bytes fields +//! on proto3. + use googletest::prelude::*; +use protobuf::prelude::*; + use protobuf::Optional; use unittest_proto3_optional_rust_proto::{test_proto3_optional, TestProto3Optional}; use unittest_proto3_rust_proto::{test_all_types, TestAllTypes}; @@ -251,3 +254,29 @@ fn test_ctype_stringpiece() { msg.set_optional_string_piece("hello"); assert_that!(msg.optional_string_piece(), eq("hello")); } + +#[googletest::test] +fn test_msg_clear() { + let mut m = TestAllTypes::new(); + m.set_optional_int32(42); + assert_that!(m.optional_int32(), eq(42)); + m.clear(); + assert_that!(m.optional_int32(), eq(0)); +} + +#[googletest::test] +fn test_submsg_clear() { + let mut m = TestAllTypes::new(); + let mut sub = m.optional_nested_message_mut(); + sub.set_bb(7); + + assert_that!(m.has_optional_nested_message(), eq(true)); + assert_that!(m.optional_nested_message().bb(), eq(7)); + + m.optional_nested_message_mut().clear(); + + // .clear() on the submsg doesn't affect its presence on the parent: + assert_that!(m.has_optional_nested_message(), eq(true)); + // ...but it does clear the submsg's value: + assert_that!(m.optional_nested_message().bb(), eq(0)); +} diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index 2a7795ec06..af020f8ba4 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -8,6 +8,8 @@ //! Tests covering accessors for singular bool, int32, int64, and bytes fields. use googletest::prelude::*; +use protobuf::prelude::*; + use protobuf::{Optional, ProtoBytes, ProtoStr, ProtoString}; use std::borrow::Cow; use std::ffi::OsString; @@ -917,3 +919,29 @@ fn test_ctype_stringpiece() { assert_that!(msg.optional_string_piece(), eq("hello")); assert_that!(msg.has_optional_string_piece(), eq(true)); } + +#[googletest::test] +fn test_msg_clear() { + let mut m = TestAllTypes::new(); + m.set_optional_int32(42); + assert_that!(m.has_optional_int32(), eq(true)); + m.clear(); + assert_that!(m.has_optional_int32(), eq(false)); +} + +#[googletest::test] +fn test_submsg_clear() { + let mut m = TestAllTypes::new(); + let mut sub = m.optional_nested_message_mut(); + sub.set_bb(7); + + assert_that!(m.has_optional_nested_message(), eq(true)); + assert_that!(m.optional_nested_message().bb(), eq(7)); + + m.optional_nested_message_mut().clear(); + + // .clear() on the submsg doesn't affect its presence on the parent: + assert_that!(m.has_optional_nested_message(), eq(true)); + // ...but it does clear the submsg's value: + assert_that!(m.optional_nested_message().bb(), eq(0)); +} diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 67319afe6d..3511918b3c 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -32,8 +32,9 @@ pub use map::{ mod message; pub use message::{ - upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_IsEqual, - upb_Message_MergeFrom, upb_Message_New, upb_Message_SetBaseField, RawMessage, + upb_Message, upb_Message_Clear, upb_Message_DeepClone, upb_Message_DeepCopy, + upb_Message_IsEqual, upb_Message_MergeFrom, upb_Message_New, upb_Message_SetBaseField, + RawMessage, }; mod message_value; diff --git a/rust/upb/message.rs b/rust/upb/message.rs index 67d1e06e78..b78e495808 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -13,10 +13,20 @@ opaque_pointee!(upb_Message); pub type RawMessage = NonNull; extern "C" { - /// SAFETY: No constraints. + /// SAFETY: + /// - `mini_table` and `arena` must be valid to deref pub fn upb_Message_New(mini_table: *const upb_MiniTable, arena: RawArena) -> Option; + /// SAFETY: + /// - `m` and `mini_table` must be valid to deref + /// - `mini_table` must be the MiniTable associtaed with `m` + pub fn upb_Message_Clear(m: RawMessage, mini_table: *const upb_MiniTable); + + /// SAFETY: + /// - All four arguments must be valid to deref + /// - `mini_table` must be the MiniTable associated with both `dst` and + /// `src` pub fn upb_Message_DeepCopy( dst: RawMessage, src: RawMessage, @@ -24,18 +34,29 @@ extern "C" { arena: RawArena, ); + /// SAFETY: + /// - All three arguments must be valid to deref + /// - `mini_table` must be the MiniTable associated with `m` pub fn upb_Message_DeepClone( m: RawMessage, mini_table: *const upb_MiniTable, arena: RawArena, ) -> Option; + /// SAFETY: + /// - `m` and `mini_table` must be valid to deref + /// - `mini_table` must be the MiniTable associated with `m` + /// - `val` must be a pointer to legally readable memory of the correct type + /// for the field described by `mini_table` pub fn upb_Message_SetBaseField( m: RawMessage, mini_table: *const upb_MiniTableField, val: *const std::ffi::c_void, ); + /// SAFETY: + /// - All four arguments must be valid to deref + /// - `mini_table` must be the MiniTable associated with both `m1` and `m2` pub fn upb_Message_IsEqual( m1: RawMessage, m2: RawMessage, @@ -43,6 +64,11 @@ extern "C" { options: i32, ) -> bool; + /// SAFETY: + /// - `dst`, `src`, `mini_table` and `arena` must be valid to deref + /// - `extreg` must be valid to deref or nullptr + /// - `mini_table` must be the MiniTable associated with both `dst` and + /// `src` pub fn upb_Message_MergeFrom( dst: RawMessage, src: RawMessage, diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index a6dca4b460..c3c5013319 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -92,6 +92,28 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { ABSL_LOG(FATAL) << "unreachable"; } +void MessageMutClear(Context& ctx, const Descriptor& msg) { + switch (ctx.opts().kernel) { + case Kernel::kCpp: + ctx.Emit({{"clear_thunk", ThunkName(ctx, msg, "clear")}}, + R"rs( + unsafe { $clear_thunk$(self.raw_msg()) } + )rs"); + return; + case Kernel::kUpb: + ctx.Emit( + { + {"minitable", UpbMinitableName(msg)}, + }, + R"rs( + unsafe { + $pbr$::upb_Message_Clear(self.raw_msg(), $std$::ptr::addr_of!($minitable$)) + } + )rs"); + return; + } +} + void MessageClearAndParse(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: @@ -180,6 +202,7 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { { {"new_thunk", ThunkName(ctx, msg, "new")}, {"delete_thunk", ThunkName(ctx, msg, "delete")}, + {"clear_thunk", ThunkName(ctx, msg, "clear")}, {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, {"parse_thunk", ThunkName(ctx, msg, "parse")}, {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, @@ -200,6 +223,7 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { R"rs( fn $new_thunk$() -> $pbr$::RawMessage; fn $delete_thunk$(raw_msg: $pbr$::RawMessage); + fn $clear_thunk$(raw_msg: $pbr$::RawMessage); fn $serialize_thunk$(raw_msg: $pbr$::RawMessage, out: &mut $pbr$::SerializedData) -> bool; fn $parse_thunk$(raw_msg: $pbr$::RawMessage, data: $pbr$::SerializedData) -> bool; fn $copy_from_thunk$(dst: $pbr$::RawMessage, src: $pbr$::RawMessage); @@ -824,6 +848,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { {{"Msg", RsSafeName(msg.name())}, {"Msg::new", [&] { MessageNew(ctx, msg); }}, {"Msg::serialize", [&] { MessageSerialize(ctx, msg); }}, + {"MsgMut::clear", [&] { MessageMutClear(ctx, msg); }}, {"Msg::clear_and_parse", [&] { MessageClearAndParse(ctx, msg); }}, {"Msg::drop", [&] { MessageDrop(ctx, msg); }}, {"Msg::debug", [&] { MessageDebug(ctx, msg); }}, @@ -977,6 +1002,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::Clear for $Msg$ { + fn clear(&mut self) { + self.as_mut().clear() + } + } + impl $pb$::ClearAndParse for $Msg$ { fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { self.clear_and_parse(data) @@ -1096,6 +1127,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::Clear for $Msg$Mut<'_> { + fn clear(&mut self) { + $MsgMut::clear$ + } + } + #[allow(dead_code)] impl<'msg> $Msg$Mut<'msg> { #[doc(hidden)] @@ -1312,6 +1349,7 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { {"QualifiedMsg", cpp::QualifiedClassName(&msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, {"delete_thunk", ThunkName(ctx, msg, "delete")}, + {"clear_thunk", ThunkName(ctx, msg, "clear")}, {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, {"parse_thunk", ThunkName(ctx, msg, "parse")}, {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, @@ -1354,6 +1392,9 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { extern $abi$ { void* $new_thunk$() { return new $QualifiedMsg$(); } void $delete_thunk$(void* ptr) { delete static_cast<$QualifiedMsg$*>(ptr); } + void $clear_thunk$(void* ptr) { + static_cast<$QualifiedMsg$*>(ptr)->Clear(); + } bool $serialize_thunk$($QualifiedMsg$* msg, google::protobuf::rust::SerializedData* out) { return google::protobuf::rust::SerializeMsg(msg, out); } From 1e1bf0d1fc603c6586076c9e2f92cc7931a1f83d Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Fri, 26 Jul 2024 11:07:22 -0700 Subject: [PATCH 091/107] Move the declaration of the upb map begin constant into rust PiperOrigin-RevId: 656463679 --- rust/upb.rs | 3 +-- rust/upb/lib.rs | 2 +- rust/upb/map.rs | 4 ++-- rust/upb/upb_api.c | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/rust/upb.rs b/rust/upb.rs index eda6fc4e27..b90de06d41 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -682,8 +682,7 @@ pub struct RawMapIter { impl RawMapIter { pub fn new(_private: Private, map: RawMap) -> Self { - // SAFETY: __rust_proto_kUpb_Map_Begin is never modified - RawMapIter { map, iter: unsafe { __rust_proto_kUpb_Map_Begin } } + RawMapIter { map, iter: UPB_MAP_BEGIN } } /// # Safety diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 3511918b3c..84b09fc5de 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -27,7 +27,7 @@ pub use extension_registry::{upb_ExtensionRegistry, RawExtensionRegistry}; mod map; pub use map::{ upb_Map, upb_Map_Clear, upb_Map_Delete, upb_Map_Get, upb_Map_Insert, upb_Map_New, upb_Map_Next, - upb_Map_Size, MapInsertStatus, RawMap, __rust_proto_kUpb_Map_Begin, + upb_Map_Size, MapInsertStatus, RawMap, UPB_MAP_BEGIN, }; mod message; diff --git a/rust/upb/map.rs b/rust/upb/map.rs index 5b60149a74..5b3a8d7bbb 100644 --- a/rust/upb/map.rs +++ b/rust/upb/map.rs @@ -21,9 +21,9 @@ pub enum MapInsertStatus { OutOfMemory = 2, } -extern "C" { - pub static __rust_proto_kUpb_Map_Begin: usize; +pub const UPB_MAP_BEGIN: usize = usize::MAX; +extern "C" { pub fn upb_Map_New(arena: RawArena, key_type: CType, value_type: CType) -> RawMap; pub fn upb_Map_Size(map: RawMap) -> usize; pub fn upb_Map_Insert( diff --git a/rust/upb/upb_api.c b/rust/upb/upb_api.c index 16cc9e081c..2b23a52e17 100644 --- a/rust/upb/upb_api.c +++ b/rust/upb/upb_api.c @@ -21,5 +21,3 @@ #include "upb/mini_table/message.h" // IWYU pragma: keep #include "upb/text/debug_string.h" // IWYU pragma: keep // go/keep-sorted end - -const size_t __rust_proto_kUpb_Map_Begin = kUpb_Map_Begin; From 7141c304251619a8d5c690e47e0c6e5d67dabe30 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 26 Jul 2024 11:13:39 -0700 Subject: [PATCH 092/107] Rust: update map setters to take `IntoProxied` I also added a blanket implementation of `IntoProxied for T` so that we don't have to duplicate this no-op implementation for all our types. PiperOrigin-RevId: 656465755 --- rust/map.rs | 24 +++++++ rust/primitive.rs | 9 +-- rust/proxied.rs | 6 ++ rust/repeated.rs | 9 --- rust/string.rs | 12 ---- rust/test/shared/BUILD | 8 +++ rust/test/shared/accessors_map_test.rs | 65 +++++++++++++++---- .../protobuf/compiler/rust/accessors/map.cc | 6 +- src/google/protobuf/compiler/rust/enum.cc | 6 -- src/google/protobuf/compiler/rust/message.cc | 12 ---- 10 files changed, 95 insertions(+), 62 deletions(-) diff --git a/rust/map.rs b/rust/map.rs index f136614830..57647d649b 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -374,6 +374,30 @@ where } } +impl<'msg, K, V> IntoProxied> for MapView<'msg, K, V> +where + K: Proxied, + V: ProxiedInMapValue, + View<'msg, V>: IntoProxied, +{ + fn into_proxied(self, _private: Private) -> Map { + let mut m = Map::::new(); + m.as_mut().copy_from(self); + m + } +} + +impl<'msg, K, V> IntoProxied> for MapMut<'msg, K, V> +where + K: Proxied, + V: ProxiedInMapValue, + View<'msg, V>: IntoProxied, +{ + fn into_proxied(self, _private: Private) -> Map { + self.into_view().into_proxied(Private) + } +} + /// An iterator visiting all key-value pairs in arbitrary order. /// /// The iterator element type is `(View, View)`. diff --git a/rust/primitive.rs b/rust/primitive.rs index 06c90f1536..a87f3305f0 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -4,8 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::__internal::Private; -use crate::{AsView, IntoProxied, IntoView, Proxied, Proxy, ViewProxy}; +use crate::{AsView, IntoView, Proxied, Proxy, ViewProxy}; macro_rules! impl_singular_primitives { ($($t:ty),*) => { @@ -35,12 +34,6 @@ macro_rules! impl_singular_primitives { impl<'msg> ViewProxy<'msg> for $t {} - impl IntoProxied<$t> for $t { - fn into_proxied(self, _private: Private) -> $t { - self - } - } - // ProxiedInRepeated is implemented in {cpp,upb}.rs )* } diff --git a/rust/proxied.rs b/rust/proxied.rs index 0267b5fc71..c56b18544f 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -239,6 +239,12 @@ pub trait IntoProxied { fn into_proxied(self, _private: Private) -> T; } +impl IntoProxied for T { + fn into_proxied(self, _private: Private) -> T { + self + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/repeated.rs b/rust/repeated.rs index 972209bec2..1a43dceab0 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -233,15 +233,6 @@ where } } -impl IntoProxied> for Repeated -where - T: ProxiedInRepeated, -{ - fn into_proxied(self, _private: Private) -> Repeated { - self - } -} - impl<'msg, T> IntoProxied> for RepeatedView<'msg, T> where T: 'msg + ProxiedInRepeated, diff --git a/rust/string.rs b/rust/string.rs index b25e548392..321e035acb 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -79,12 +79,6 @@ impl AsView for ProtoBytes { } } -impl IntoProxied for ProtoBytes { - fn into_proxied(self, _private: Private) -> ProtoBytes { - self - } -} - impl IntoProxied for &[u8] { fn into_proxied(self, _private: Private) -> ProtoBytes { ProtoBytes::from(self) @@ -238,12 +232,6 @@ impl From<&[u8]> for ProtoString { } } -impl IntoProxied for ProtoString { - fn into_proxied(self, _private: Private) -> ProtoString { - self - } -} - impl IntoProxied for &str { fn into_proxied(self, _private: Private) -> ProtoString { ProtoString::from(self) diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index ecda92d474..4c789cdfba 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -367,10 +367,14 @@ rust_test( rust_test( name = "accessors_map_cpp_test", srcs = ["accessors_map_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_cpp_export", "//rust/test:enums_cpp_rust_proto", "//rust/test:map_unittest_cpp_rust_proto", "//rust/test:unittest_cpp_rust_proto", @@ -381,10 +385,14 @@ rust_test( rust_test( name = "accessors_map_upb_test", srcs = ["accessors_map_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_upb_export", "//rust/test:enums_upb_rust_proto", "//rust/test:map_unittest_upb_rust_proto", "//rust/test:unittest_upb_rust_proto", diff --git a/rust/test/shared/accessors_map_test.rs b/rust/test/shared/accessors_map_test.rs index b284a24752..4ec026eb1f 100644 --- a/rust/test/shared/accessors_map_test.rs +++ b/rust/test/shared/accessors_map_test.rs @@ -9,6 +9,7 @@ use enums_rust_proto::{test_map_with_nested_enum, TestMapWithNestedEnum}; use googletest::prelude::*; use map_unittest_rust_proto::{MapEnum, TestMap, TestMapWithMessages}; use paste::paste; +use protobuf::ProtoString; use std::collections::HashMap; use unittest_rust_proto::TestAllTypes; @@ -156,19 +157,59 @@ fn test_bytes_and_string_copied() { #[googletest::test] fn test_map_setter() { - let mut msg = TestMap::new(); - msg.map_string_string_mut().insert("hello", "world"); - msg.map_string_string_mut().insert("fizz", "buzz"); + // Set Map + { + let mut msg = TestMap::new(); + let mut map = protobuf::Map::::new(); + map.as_mut().copy_from([("hello", "world"), ("fizz", "buzz")]); + msg.set_map_string_string(map); + assert_that!( + msg.map_string_string(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + } - let mut msg2 = TestMap::new(); - msg2.set_map_string_string(msg.map_string_string()); - assert_that!( - msg2.map_string_string(), - unordered_elements_are![ - eq(("hello".into(), "world".into())), - eq(("fizz".into(), "buzz".into())) - ] - ); + // Set MapView + { + let mut msg = TestMap::new(); + let mut map = protobuf::Map::::new(); + map.as_mut().copy_from([("hello", "world"), ("fizz", "buzz")]); + msg.set_map_string_string(map.as_view()); + assert_that!( + msg.map_string_string(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + } + + // Set MapMut + { + let mut msg = TestMap::new(); + let mut map = protobuf::Map::::new(); + map.as_mut().copy_from([("hello", "world"), ("fizz", "buzz")]); + msg.set_map_string_string(map.as_mut()); + assert_that!( + msg.map_string_string(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + + // The original map should remain unchanged. + assert_that!( + map.as_view(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + } } macro_rules! generate_map_with_msg_values_tests { diff --git a/src/google/protobuf/compiler/rust/accessors/map.cc b/src/google/protobuf/compiler/rust/accessors/map.cc index e38f03da5b..2bd16ff79c 100644 --- a/src/google/protobuf/compiler/rust/accessors/map.cc +++ b/src/google/protobuf/compiler/rust/accessors/map.cc @@ -107,9 +107,9 @@ void Map::InMsgImpl(Context& ctx, const FieldDescriptor& field, return; } ctx.Emit({}, R"rs( - pub fn set_$raw_field_name$(&mut self, src: $pb$::MapView<'_, $Key$, $Value$>) { - // TODO: Implement IntoProxied and avoid copying. - self.$field$_mut().copy_from(src); + pub fn set_$raw_field_name$(&mut self, src: impl $pb$::IntoProxied<$pb$::Map<$Key$, $Value$>>) { + // TODO: b/355493062 - Fix this extra copy. + self.$field$_mut().copy_from(src.into_proxied($pbi$::Private).as_view()); } )rs"); }}}, diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 9513c93eb2..8c39cb8602 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -389,12 +389,6 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } } - impl $pb$::IntoProxied<$name$> for $name$ { - fn into_proxied(self, _: $pbi$::Private) -> Self { - self - } - } - impl $pb$::IntoProxied for $name$ { fn into_proxied(self, _: $pbi$::Private) -> i32 { self.0 diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index c3c5013319..dd7369f299 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -287,12 +287,6 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { $pb$::IntoProxied::into_proxied($pb$::IntoView::into_view(self), _private) } } - - impl $pb$::IntoProxied<$Msg$> for $Msg$ { - fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - self - } - } )rs"); return; @@ -316,12 +310,6 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { $pb$::IntoProxied::into_proxied($pb$::IntoView::into_view(self), _private) } } - - impl $pb$::IntoProxied<$Msg$> for $Msg$ { - fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - self - } - } )rs"); return; } From f396f7cb9fc6d8fcd897f3d88c05721642db6b46 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 26 Jul 2024 11:41:38 -0700 Subject: [PATCH 093/107] Replace preprocessor macros with `ToHost` from endian PiperOrigin-RevId: 656475785 --- conformance/BUILD.bazel | 1 + src/google/protobuf/BUILD.bazel | 18 +++++++++++++++++- src/google/protobuf/io/BUILD.bazel | 1 + src/google/protobuf/io/coded_stream.h | 27 +++------------------------ 4 files changed, 22 insertions(+), 25 deletions(-) diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index 5a6c271856..e402498e37 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -143,6 +143,7 @@ cc_library( ":conformance_cc_proto", "//src/google/protobuf", "//src/google/protobuf:descriptor_legacy", + "//src/google/protobuf:endian", "//src/google/protobuf:protobuf_lite", "//src/google/protobuf/util:differencer", "//src/google/protobuf/util:json_util", diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 38c088e092..0321694748 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -342,7 +342,7 @@ cc_library( "//src/google/protobuf:__subpackages__", ], deps = [ - "//src/google/protobuf:port", + ":port", "//src/google/protobuf/stubs:lite", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_check", @@ -479,6 +479,7 @@ cc_library( "repeated_ptr_field.cc", "wire_format_lite.cc", ], + # TODO Fix ODR violations across BUILD.bazel files. hdrs = [ "any.h", "arena.h", @@ -527,6 +528,7 @@ cc_library( ":arena_align", ":arena_allocation_policy", ":arena_cleanup", + ":endian", ":internal_visibility", ":port", ":string_block", @@ -2066,3 +2068,17 @@ filegroup( ], visibility = ["//pkg:__pkg__"], ) + +cc_library( + name = "endian", + hdrs = ["endian.h"], + strip_include_prefix = "/src", + visibility = [ + "//conformance:__subpackages__", + "//src/google/protobuf:__subpackages__", + ], + deps = [ + ":port", + "@com_google_absl//absl/base:config", + ], +) diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel index 096ebf71b8..e003c65f89 100644 --- a/src/google/protobuf/io/BUILD.bazel +++ b/src/google/protobuf/io/BUILD.bazel @@ -27,6 +27,7 @@ cc_library( deps = [ ":io_win32", "//src/google/protobuf:arena", + "//src/google/protobuf:endian", "//src/google/protobuf:port", "//src/google/protobuf/stubs:lite", "@com_google_absl//absl/base", diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 4a5ca80cf8..3a40dedd80 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -111,9 +111,9 @@ #include "absl/numeric/bits.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "google/protobuf/endian.h" #include "google/protobuf/port.h" - // Must be included last. #include "google/protobuf/port_def.inc" @@ -1324,37 +1324,16 @@ inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) { // static inline const uint8_t* CodedInputStream::ReadLittleEndian32FromArray( const uint8_t* buffer, uint32_t* value) { -#if defined(ABSL_IS_LITTLE_ENDIAN) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(value, buffer, sizeof(*value)); + *value = google::protobuf::internal::little_endian::ToHost(*value); return buffer + sizeof(*value); -#else - *value = (static_cast(buffer[0])) | - (static_cast(buffer[1]) << 8) | - (static_cast(buffer[2]) << 16) | - (static_cast(buffer[3]) << 24); - return buffer + sizeof(*value); -#endif } // static inline const uint8_t* CodedInputStream::ReadLittleEndian64FromArray( const uint8_t* buffer, uint64_t* value) { -#if defined(ABSL_IS_LITTLE_ENDIAN) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(value, buffer, sizeof(*value)); + *value = google::protobuf::internal::little_endian::ToHost(*value); return buffer + sizeof(*value); -#else - uint32_t part0 = (static_cast(buffer[0])) | - (static_cast(buffer[1]) << 8) | - (static_cast(buffer[2]) << 16) | - (static_cast(buffer[3]) << 24); - uint32_t part1 = (static_cast(buffer[4])) | - (static_cast(buffer[5]) << 8) | - (static_cast(buffer[6]) << 16) | - (static_cast(buffer[7]) << 24); - *value = static_cast(part0) | (static_cast(part1) << 32); - return buffer + sizeof(*value); -#endif } inline bool CodedInputStream::ReadLittleEndian32(uint32_t* value) { From 50f58a6e2008c70ac77e8b2e476ff9bff8393a67 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 26 Jul 2024 12:47:58 -0700 Subject: [PATCH 094/107] Migrate to standard parameterized test framework PiperOrigin-RevId: 656497777 --- src/google/protobuf/io/BUILD.bazel | 1 + src/google/protobuf/io/coded_stream.cc | 5 +- .../protobuf/io/coded_stream_unittest.cc | 358 ++++++++++++------ 3 files changed, 250 insertions(+), 114 deletions(-) diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel index e003c65f89..b22c677358 100644 --- a/src/google/protobuf/io/BUILD.bazel +++ b/src/google/protobuf/io/BUILD.bazel @@ -209,6 +209,7 @@ cc_test( "//src/google/protobuf/testing", "//src/google/protobuf/testing:file", "@com_google_absl//absl/base", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:log_severity", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc index a9297e116f..23a692a0bc 100644 --- a/src/google/protobuf/io/coded_stream.cc +++ b/src/google/protobuf/io/coded_stream.cc @@ -20,9 +20,13 @@ #include #include +#include #include #include #include +#include +#include +#include #include #include "absl/log/absl_check.h" @@ -33,7 +37,6 @@ #include "google/protobuf/arena.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" -#include "google/protobuf/port.h" // Must be included last. diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc index 8c7b65c961..295bcbf1b9 100644 --- a/src/google/protobuf/io/coded_stream_unittest.cc +++ b/src/google/protobuf/io/coded_stream_unittest.cc @@ -16,20 +16,29 @@ #include #include +#include +#include +#include +#include +#include #include +#include #include +#include +#include #include -#include "google/protobuf/stubs/common.h" +#include #include #include "absl/base/casts.h" #include "absl/base/log_severity.h" -#include "absl/log/absl_check.h" +#include "absl/base/macros.h" #include "absl/log/absl_log.h" #include "absl/log/scoped_mock_log.h" #include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/testing/googletest.h" @@ -42,72 +51,6 @@ namespace protobuf { namespace io { namespace { -// =================================================================== -// Data-Driven Test Infrastructure - -// TEST_1D and TEST_2D are macros I'd eventually like to see added to -// gTest. These macros can be used to declare tests which should be -// run multiple times, once for each item in some input array. TEST_1D -// tests all cases in a single input array. TEST_2D tests all -// combinations of cases from two arrays. The arrays must be statically -// defined such that the ABSL_ARRAYSIZE() macro works on them. Example: -// -// int kCases[] = {1, 2, 3, 4} -// TEST_1D(MyFixture, MyTest, kCases) { -// EXPECT_GT(kCases_case, 0); -// } -// -// This test iterates through the numbers 1, 2, 3, and 4 and tests that -// they are all grater than zero. In case of failure, the exact case -// which failed will be printed. The case type must be printable using -// ostream::operator<<. - -// TODO: gTest now supports "parameterized tests" which would be -// a better way to accomplish this. Rewrite when time permits. - -#define TEST_1D(FIXTURE, NAME, CASES) \ - class FIXTURE##_##NAME##_DD : public FIXTURE { \ - protected: \ - template \ - void DoSingleCase(const CaseType& CASES##_case); \ - }; \ - \ - TEST_F(FIXTURE##_##NAME##_DD, NAME) { \ - for (size_t i = 0; i < ABSL_ARRAYSIZE(CASES); i++) { \ - SCOPED_TRACE(testing::Message() \ - << #CASES " case #" << i << ": " << CASES[i]); \ - DoSingleCase(CASES[i]); \ - } \ - } \ - \ - template \ - void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType& CASES##_case) - -#define TEST_2D(FIXTURE, NAME, CASES1, CASES2) \ - class FIXTURE##_##NAME##_DD : public FIXTURE { \ - protected: \ - template \ - void DoSingleCase(const CaseType1& CASES1##_case, \ - const CaseType2& CASES2##_case); \ - }; \ - \ - TEST_F(FIXTURE##_##NAME##_DD, NAME) { \ - for (size_t i = 0; i < ABSL_ARRAYSIZE(CASES1); i++) { \ - for (size_t j = 0; j < ABSL_ARRAYSIZE(CASES2); j++) { \ - SCOPED_TRACE(testing::Message() \ - << #CASES1 " case #" << i << ": " << CASES1[i] << ", " \ - << #CASES2 " case #" << j << ": " << CASES2[j]); \ - DoSingleCase(CASES1[i], CASES2[j]); \ - } \ - } \ - } \ - \ - template \ - void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType1& CASES1##_case, \ - const CaseType2& CASES2##_case) - -// =================================================================== - class CodedStreamTest : public testing::Test { protected: // Buffer used during most of the tests. This assumes tests run sequentially. @@ -171,7 +114,13 @@ VarintCase kVarintCases[] = { (uint64_t{0x26u} << 56) | (uint64_t{0x01u} << 63)}, }; -TEST_2D(CodedStreamTest, ReadVarint32, kVarintCases, kBlockSizes) { +class VarintCasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +TEST_P(VarintCasesWithSizes, ReadVarint32) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -186,7 +135,9 @@ TEST_2D(CodedStreamTest, ReadVarint32, kVarintCases, kBlockSizes) { EXPECT_EQ(kVarintCases_case.size, input.ByteCount()); } -TEST_2D(CodedStreamTest, ReadTag, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, ReadTag) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -231,7 +182,11 @@ TEST_F(CodedStreamTest, EmptyInputBeforeEos) { EXPECT_TRUE(input.ConsumedEntireMessage()); } -TEST_1D(CodedStreamTest, ExpectTag, kVarintCases) { +class VarintCases : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(VarintCases, ExpectTag) { + const VarintCase& kVarintCases_case = GetParam(); // Leave one byte at the beginning of the buffer so we can read it // to force the first buffer to be loaded. buffer_[0] = '\0'; @@ -265,7 +220,8 @@ TEST_1D(CodedStreamTest, ExpectTag, kVarintCases) { } } -TEST_1D(CodedStreamTest, ExpectTagFromArray, kVarintCases) { +TEST_P(VarintCases, ExpectTagFromArray) { + const VarintCase& kVarintCases_case = GetParam(); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); const uint32_t expected_value = @@ -283,7 +239,9 @@ TEST_1D(CodedStreamTest, ExpectTagFromArray, kVarintCases) { } } -TEST_2D(CodedStreamTest, ReadVarint64, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, ReadVarint64) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -298,7 +256,9 @@ TEST_2D(CodedStreamTest, ReadVarint64, kVarintCases, kBlockSizes) { EXPECT_EQ(kVarintCases_case.size, input.ByteCount()); } -TEST_2D(CodedStreamTest, WriteVarint32, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, WriteVarint32) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); if (kVarintCases_case.value > uint64_t{0x00000000FFFFFFFFu}) { // Skip this test for the 64-bit values. return; @@ -320,7 +280,9 @@ TEST_2D(CodedStreamTest, WriteVarint32, kVarintCases, kBlockSizes) { memcmp(buffer_, kVarintCases_case.bytes, kVarintCases_case.size)); } -TEST_2D(CodedStreamTest, WriteVarint64, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, WriteVarint64) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -337,11 +299,15 @@ TEST_2D(CodedStreamTest, WriteVarint64, kVarintCases, kBlockSizes) { memcmp(buffer_, kVarintCases_case.bytes, kVarintCases_case.size)); } +class SignedVarintCasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; int32_t kSignExtendedVarintCases[] = {0, 1, -1, 1237894, -37895138}; -TEST_2D(CodedStreamTest, WriteVarint32SignExtended, kSignExtendedVarintCases, - kBlockSizes) { +TEST_P(SignedVarintCasesWithSizes, WriteVarint32SignExtended) { + const int32_t& kSignExtendedVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -386,6 +352,7 @@ struct VarintErrorCase { uint8_t bytes[12]; size_t size; bool can_parse; + const char* name; }; inline std::ostream& operator<<(std::ostream& os, const VarintErrorCase& c) { @@ -395,24 +362,34 @@ inline std::ostream& operator<<(std::ostream& os, const VarintErrorCase& c) { const VarintErrorCase kVarintErrorCases[] = { // Control case. (Insures that there isn't something else wrong that // makes parsing always fail.) - {{0x00}, 1, true}, + {{0x00}, 1, true, "Control"}, // No input data. - {{}, 0, false}, + {{}, 0, false, "No_Input"}, // Input ends unexpectedly. - {{0xf0, 0xab}, 2, false}, + {{0xf0, 0xab}, 2, false, "Input_Ends_Unexpectedly"}, // Input ends unexpectedly after 32 bits. - {{0xf0, 0xab, 0xc9, 0x9a, 0xf8, 0xb2}, 6, false}, + {{0xf0, 0xab, 0xc9, 0x9a, 0xf8, 0xb2}, + 6, + false, + "Input_Ends_Unexpectedly_After_32_Bits"}, // Longer than 10 bytes. {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}, 11, - false}, + false, + "Longer_Than_10_Bytes"}, }; -TEST_2D(CodedStreamTest, ReadVarint32Error, kVarintErrorCases, kBlockSizes) { +class VarintErrorCasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +TEST_P(VarintErrorCasesWithSizes, ReadVarint32Error) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -422,8 +399,10 @@ TEST_2D(CodedStreamTest, ReadVarint32Error, kVarintErrorCases, kBlockSizes) { EXPECT_EQ(kVarintErrorCases_case.can_parse, coded_input.ReadVarint32(&value)); } -TEST_2D(CodedStreamTest, ReadVarint32Error_LeavesValueInInitializedState, - kVarintErrorCases, kBlockSizes) { +TEST_P(VarintErrorCasesWithSizes, + ReadVarint32Error_LeavesValueInInitializedState) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -437,7 +416,9 @@ TEST_2D(CodedStreamTest, ReadVarint32Error_LeavesValueInInitializedState, EXPECT_EQ(value, value); } -TEST_2D(CodedStreamTest, ReadVarint64Error, kVarintErrorCases, kBlockSizes) { +TEST_P(VarintErrorCasesWithSizes, ReadVarint64Error) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -447,8 +428,10 @@ TEST_2D(CodedStreamTest, ReadVarint64Error, kVarintErrorCases, kBlockSizes) { EXPECT_EQ(kVarintErrorCases_case.can_parse, coded_input.ReadVarint64(&value)); } -TEST_2D(CodedStreamTest, ReadVarint64Error_LeavesValueInInitializedState, - kVarintErrorCases, kBlockSizes) { +TEST_P(VarintErrorCasesWithSizes, + ReadVarint64Error_LeavesValueInInitializedState) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -485,7 +468,11 @@ VarintSizeCase kVarintSizeCases[] = { {uint64_t{11964378330978735131u}, 10}, }; -TEST_1D(CodedStreamTest, VarintSize32, kVarintSizeCases) { +class VarintSizeCases : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(VarintSizeCases, VarintSize32) { + VarintSizeCase kVarintSizeCases_case = GetParam(); if (kVarintSizeCases_case.value > 0xffffffffu) { // Skip 64-bit values. return; @@ -496,7 +483,8 @@ TEST_1D(CodedStreamTest, VarintSize32, kVarintSizeCases) { static_cast(kVarintSizeCases_case.value))); } -TEST_1D(CodedStreamTest, VarintSize64, kVarintSizeCases) { +TEST_P(VarintSizeCases, VarintSize64) { + VarintSizeCase kVarintSizeCases_case = GetParam(); EXPECT_EQ(kVarintSizeCases_case.size, CodedOutputStream::VarintSize64(kVarintSizeCases_case.value)); } @@ -535,6 +523,20 @@ struct Fixed64Case { uint64_t value; // Parsed value. }; +class Fixed32Cases : public CodedStreamTest, + public testing::WithParamInterface {}; + +class Fixed32CasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +class Fixed64Cases : public CodedStreamTest, + public testing::WithParamInterface {}; + +class Fixed64CasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + inline std::ostream& operator<<(std::ostream& os, const Fixed32Case& c) { return os << "0x" << std::hex << c.value << std::dec; } @@ -555,7 +557,9 @@ Fixed64Case kFixed64Cases[] = { uint64_t{0x8877665544332211u}}, }; -TEST_2D(CodedStreamTest, ReadLittleEndian32, kFixed32Cases, kBlockSizes) { +TEST_P(Fixed32CasesWithSizes, ReadLittleEndian32) { + Fixed32Case kFixed32Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kFixed32Cases_case.bytes, sizeof(kFixed32Cases_case.bytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -570,7 +574,9 @@ TEST_2D(CodedStreamTest, ReadLittleEndian32, kFixed32Cases, kBlockSizes) { EXPECT_EQ(sizeof(uint32_t), input.ByteCount()); } -TEST_2D(CodedStreamTest, ReadLittleEndian64, kFixed64Cases, kBlockSizes) { +TEST_P(Fixed64CasesWithSizes, ReadLittleEndian64) { + Fixed64Case kFixed64Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kFixed64Cases_case.bytes, sizeof(kFixed64Cases_case.bytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -585,7 +591,9 @@ TEST_2D(CodedStreamTest, ReadLittleEndian64, kFixed64Cases, kBlockSizes) { EXPECT_EQ(sizeof(uint64_t), input.ByteCount()); } -TEST_2D(CodedStreamTest, WriteLittleEndian32, kFixed32Cases, kBlockSizes) { +TEST_P(Fixed32CasesWithSizes, WriteLittleEndian32) { + Fixed32Case kFixed32Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -601,7 +609,9 @@ TEST_2D(CodedStreamTest, WriteLittleEndian32, kFixed32Cases, kBlockSizes) { EXPECT_EQ(0, memcmp(buffer_, kFixed32Cases_case.bytes, sizeof(uint32_t))); } -TEST_2D(CodedStreamTest, WriteLittleEndian64, kFixed64Cases, kBlockSizes) { +TEST_P(Fixed64CasesWithSizes, WriteLittleEndian64) { + Fixed64Case kFixed64Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -619,7 +629,8 @@ TEST_2D(CodedStreamTest, WriteLittleEndian64, kFixed64Cases, kBlockSizes) { // Tests using the static methods to read fixed-size values from raw arrays. -TEST_1D(CodedStreamTest, ReadLittleEndian32FromArray, kFixed32Cases) { +TEST_P(Fixed32Cases, ReadLittleEndian32FromArray) { + Fixed32Case kFixed32Cases_case = GetParam(); memcpy(buffer_, kFixed32Cases_case.bytes, sizeof(kFixed32Cases_case.bytes)); uint32_t value; @@ -629,7 +640,8 @@ TEST_1D(CodedStreamTest, ReadLittleEndian32FromArray, kFixed32Cases) { EXPECT_TRUE(end == buffer_ + sizeof(value)); } -TEST_1D(CodedStreamTest, ReadLittleEndian64FromArray, kFixed64Cases) { +TEST_P(Fixed64Cases, ReadLittleEndian64FromArray) { + Fixed64Case kFixed64Cases_case = GetParam(); memcpy(buffer_, kFixed64Cases_case.bytes, sizeof(kFixed64Cases_case.bytes)); uint64_t value; @@ -644,7 +656,11 @@ TEST_1D(CodedStreamTest, ReadLittleEndian64FromArray, kFixed64Cases) { const char kRawBytes[] = "Some bytes which will be written and read raw."; -TEST_1D(CodedStreamTest, ReadRaw, kBlockSizes) { +class BlockSizes : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(BlockSizes, ReadRaw) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); char read_buffer[sizeof(kRawBytes)]; @@ -659,7 +675,8 @@ TEST_1D(CodedStreamTest, ReadRaw, kBlockSizes) { EXPECT_EQ(sizeof(kRawBytes), input.ByteCount()); } -TEST_1D(CodedStreamTest, WriteRaw, kBlockSizes) { +TEST_P(BlockSizes, WriteRaw) { + int kBlockSizes_case = GetParam(); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -675,7 +692,8 @@ TEST_1D(CodedStreamTest, WriteRaw, kBlockSizes) { EXPECT_EQ(0, memcmp(buffer_, kRawBytes, sizeof(kRawBytes))); } -TEST_1D(CodedStreamTest, ReadString, kBlockSizes) { +TEST_P(BlockSizes, ReadString) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -691,7 +709,8 @@ TEST_1D(CodedStreamTest, ReadString, kBlockSizes) { } // Check to make sure ReadString doesn't crash on impossibly large strings. -TEST_1D(CodedStreamTest, ReadStringImpossiblyLarge, kBlockSizes) { +TEST_P(BlockSizes, ReadStringImpossiblyLarge) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -719,7 +738,8 @@ TEST_F(CodedStreamTest, ReadStringImpossiblyLargeFromStringOnHeap) { EXPECT_FALSE(coded_input.ReadString(&str, 1 << 30)); } -TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnTotalLimit, kBlockSizes) { +TEST_P(BlockSizes, ReadStringReservesMemoryOnTotalLimit) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -740,7 +760,8 @@ TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnTotalLimit, kBlockSizes) { EXPECT_EQ(strlen(kRawBytes), input.ByteCount()); } -TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnPushedLimit, kBlockSizes) { +TEST_P(BlockSizes, ReadStringReservesMemoryOnPushedLimit) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -908,7 +929,15 @@ TEST_F(CodedStreamTest, // ------------------------------------------------------------------- // Cord reads and writes -TEST_1D(CodedStreamTest, ReadCord, kBlockSizes) { +class BlockSizesWithResetCords + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +class ResetCords : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(BlockSizes, ReadCord) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -923,8 +952,9 @@ TEST_1D(CodedStreamTest, ReadCord, kBlockSizes) { EXPECT_EQ(strlen(kRawBytes), input.ByteCount()); } -TEST_1D(CodedStreamTest, ReadCordReuseCord, kBlockSizes) { +TEST_P(BlockSizes, ReadCordReuseCord) { ASSERT_GT(sizeof(buffer_), 1362 * sizeof(kRawBytes)); + int kBlockSizes_case = GetParam(); for (size_t i = 0; i < 1362; i++) { memcpy(buffer_ + i * sizeof(kRawBytes), kRawBytes, sizeof(kRawBytes)); } @@ -962,7 +992,9 @@ TEST_1D(CodedStreamTest, ReadCordReuseCord, kBlockSizes) { EXPECT_EQ(total_read, input.ByteCount()); } -TEST_2D(CodedStreamTest, ReadCordWithLimit, kBlockSizes, kResetCords) { +TEST_P(BlockSizesWithResetCords, ReadCordWithLimit) { + int kBlockSizes_case = std::get<0>(GetParam()); + bool kResetCords_case = std::get<1>(GetParam()); memcpy(buffer_, kRawBytes, strlen(kRawBytes)); ArrayInputStream input(buffer_, strlen(kRawBytes), kBlockSizes_case); CodedInputStream coded_input(&input); @@ -986,7 +1018,8 @@ TEST_2D(CodedStreamTest, ReadCordWithLimit, kBlockSizes, kResetCords) { EXPECT_EQ(std::string(kRawBytes + 10), std::string(cord)); } -TEST_1D(CodedStreamTest, ReadLargeCord, kResetCords) { +TEST_P(ResetCords, ReadLargeCord) { + bool kResetCords_case = GetParam(); absl::Cord large_cord; for (int i = 0; i < 1024; i++) { large_cord.Append(kRawBytes); @@ -1007,7 +1040,9 @@ TEST_1D(CodedStreamTest, ReadLargeCord, kResetCords) { } // Check to make sure ReadString doesn't crash on impossibly large strings. -TEST_2D(CodedStreamTest, ReadCordImpossiblyLarge, kBlockSizes, kResetCords) { +TEST_P(BlockSizesWithResetCords, ReadCordImpossiblyLarge) { + int kBlockSizes_case = std::get<0>(GetParam()); + bool kResetCords_case = std::get<1>(GetParam()); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1021,7 +1056,8 @@ TEST_2D(CodedStreamTest, ReadCordImpossiblyLarge, kBlockSizes, kResetCords) { } } -TEST_1D(CodedStreamTest, WriteCord, kBlockSizes) { +TEST_P(BlockSizes, WriteCord) { + int kBlockSizes_case = GetParam(); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1091,7 +1127,8 @@ TEST_F(CodedStreamTest, Trim) { const char kSkipTestBytes[] = ""; -TEST_1D(CodedStreamTest, SkipInput, kBlockSizes) { +TEST_P(BlockSizes, SkipInput) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kSkipTestBytes, sizeof(kSkipTestBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -1174,7 +1211,8 @@ TEST_F(CodedStreamTest, GetDirectBufferPointerInlineInput) { // ------------------------------------------------------------------- // Limits -TEST_1D(CodedStreamTest, BasicLimit, kBlockSizes) { +TEST_P(BlockSizes, BasicLimit) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1204,7 +1242,8 @@ TEST_1D(CodedStreamTest, BasicLimit, kBlockSizes) { // Test what happens when we push two limits where the second (top) one is // shorter. -TEST_1D(CodedStreamTest, SmallLimitOnTopOfBigLimit, kBlockSizes) { +TEST_P(BlockSizes, SmallLimitOnTopOfBigLimit) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1246,7 +1285,8 @@ TEST_1D(CodedStreamTest, SmallLimitOnTopOfBigLimit, kBlockSizes) { // Test what happens when we push two limits where the second (top) one is // longer. In this case, the top limit is shortened to match the previous // limit. -TEST_1D(CodedStreamTest, BigLimitOnTopOfSmallLimit, kBlockSizes) { +TEST_P(BlockSizes, BigLimitOnTopOfSmallLimit) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1498,6 +1538,98 @@ TEST_F(CodedStreamTest, InputOver2G) { EXPECT_EQ(INT_MAX - 512, input.backup_amount_); } +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, ResetCords, testing::ValuesIn(kResetCords), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("ResetCords_", param_info.param ? "true" : "false"); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, BlockSizesWithResetCords, + testing::Combine(testing::ValuesIn(kBlockSizes), + testing::ValuesIn(kResetCords)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("BlockSize_", std::get<0>(param_info.param), + "_ResetCords_", + std::get<1>(param_info.param) ? "true" : "false"); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintErrorCasesWithSizes, + testing::Combine(testing::ValuesIn(kVarintErrorCases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("VarintErrorCase_", + std::get<0>(param_info.param).name, "_BlockSize_", + std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintSizeCases, testing::ValuesIn(kVarintSizeCases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("VarintSizeCase_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintCases, testing::ValuesIn(kVarintCases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("VarintCase_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintCasesWithSizes, + testing::Combine(testing::ValuesIn(kVarintCases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("VarintCase_Value_", + std::get<0>(param_info.param).value, "_BlockSize_", + std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, BlockSizes, testing::ValuesIn(kBlockSizes), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("BlockSize_", param_info.param); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, SignedVarintCasesWithSizes, + testing::Combine(testing::ValuesIn(kSignExtendedVarintCases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("SignedVarintCase_Value_", + std::get<0>(param_info.param) < 0 ? "_Negative_" : "", + std::abs(std::get<0>(param_info.param)), + "_BlockSize_", std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed32Cases, testing::ValuesIn(kFixed32Cases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("Fixed32Case_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed64Cases, testing::ValuesIn(kFixed64Cases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("Fixed64Case_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed32CasesWithSizes, + testing::Combine(testing::ValuesIn(kFixed32Cases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("Fixed32Case_Value_", + std::get<0>(param_info.param).value, "_BlockSize_", + std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed64CasesWithSizes, + testing::Combine(testing::ValuesIn(kFixed64Cases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("Fixed64Case_Value_", + std::get<0>(param_info.param).value, "_BlockSize_", + std::get<1>(param_info.param)); + }); + } // namespace } // namespace io } // namespace protobuf From eceb8ccc0db3512e305c1b5a6128d7c421f62655 Mon Sep 17 00:00:00 2001 From: Yamil Morales Date: Fri, 26 Jul 2024 13:25:59 -0700 Subject: [PATCH 095/107] Implementation of go/conformanceDebuggability. Refactor ForkPipeRunner::RunTest(): There's no need for rerunning through argv for common flags between suites. This also enables "debug_test_names" to be shared between suites. PiperOrigin-RevId: 656509695 --- conformance/BUILD.bazel | 1 + conformance/binary_json_conformance_suite.cc | 22 ++- conformance/conformance_cpp.cc | 12 +- conformance/conformance_python.py | 36 +++-- conformance/conformance_test.cc | 114 ++++++++++++- conformance/conformance_test.h | 39 ++++- conformance/conformance_test_runner.cc | 162 +++++++++++++------ conformance/text_format_conformance_suite.cc | 5 +- 8 files changed, 301 insertions(+), 90 deletions(-) diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index e402498e37..e38330df62 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -233,6 +233,7 @@ cc_binary( "//src/google/protobuf", "//src/google/protobuf:port", "//src/google/protobuf:protobuf_lite", + "//src/google/protobuf/json", "//src/google/protobuf/stubs", "//src/google/protobuf/util:json_util", "//src/google/protobuf/util:type_resolver", diff --git a/conformance/binary_json_conformance_suite.cc b/conformance/binary_json_conformance_suite.cc index de575bbfdd..892a2a4426 100644 --- a/conformance/binary_json_conformance_suite.cc +++ b/conformance/binary_json_conformance_suite.cc @@ -445,7 +445,10 @@ void BinaryAndJsonConformanceSuiteImpl:: absl::StrCat(setting.ConformanceLevelToString(level), ".", setting.GetSyntaxIdentifier(), ".ProtobufInput.", test_name); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + TestStatus test; test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { @@ -641,7 +644,9 @@ void BinaryAndJsonConformanceSuiteImpl< setting.ConformanceLevelToString(level), ".", setting.GetSyntaxIdentifier(), ".JsonInput.", test_name, ".Validator"); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } TestStatus test; test.set_name(effective_test_name); @@ -689,7 +694,9 @@ void BinaryAndJsonConformanceSuiteImpl::ExpectParseFailureForJson( absl::StrCat(setting.ConformanceLevelToString(level), ".", SyntaxIdentifier(), ".JsonInput.", test_name); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } TestStatus test; test.set_name(effective_test_name); @@ -722,7 +729,9 @@ void BinaryAndJsonConformanceSuiteImpl:: absl::StrCat(setting.ConformanceLevelToString(level), ".", SyntaxIdentifier(), ".", test_name, ".JsonOutput"); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } TestStatus test; test.set_name(effective_test_name); @@ -1438,7 +1447,9 @@ void BinaryAndJsonConformanceSuiteImpl::TestUnknownOrdering() { conformance::BINARY_TEST, prototype, "UnknownOrdering", serialized); const ConformanceRequest& request = setting.GetRequest(); ConformanceResponse response; - suite_.RunTest(setting.GetTestName(), request, &response); + if (!suite_.RunTest(setting.GetTestName(), request, &response)) { + return; + } MessageType response_message; TestStatus test; @@ -1447,6 +1458,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestUnknownOrdering() { suite_.ReportSkip(test, request, response); return; } + suite_.ParseResponse(response, setting, &response_message); const UnknownFieldSet& ufs = response_message.unknown_fields(); diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 0f4196a480..45fa91aa7f 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -7,15 +7,15 @@ #include #include +#include #include #include #include +#include #include #include -#include "google/protobuf/util/json_util.h" -#include "google/protobuf/util/type_resolver_util.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" @@ -27,12 +27,15 @@ #include "editions/golden/test_messages_proto2_editions.pb.h" #include "editions/golden/test_messages_proto3_editions.pb.h" #include "google/protobuf/endian.h" +#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" #include "google/protobuf/test_messages_proto2.pb.h" #include "google/protobuf/test_messages_proto3.pb.h" #include "google/protobuf/test_messages_proto3.pb.h" #include "google/protobuf/text_format.h" +#include "google/protobuf/util/json_util.h" #include "google/protobuf/util/type_resolver.h" +#include "google/protobuf/util/type_resolver_util.h" #include "google/protobuf/stubs/status_macros.h" // Must be included last. @@ -241,8 +244,9 @@ absl::StatusOr Harness::ServeConformanceRequest() { serialized_output.size())); if (verbose_) { - ABSL_LOG(INFO) << "conformance-cpp: request=" << request.ShortDebugString() - << ", response=" << response->ShortDebugString(); + ABSL_LOG(INFO) << "conformance-cpp: request=" + << google::protobuf::ShortFormat(request) + << ", response=" << google::protobuf::ShortFormat(*response); } return false; } diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py index eea05d9fe7..77076bd885 100755 --- a/conformance/conformance_python.py +++ b/conformance/conformance_python.py @@ -102,32 +102,34 @@ def do_test(request): response.protobuf_payload = failure_set.SerializeToString() return response - isJson = (request.WhichOneof('payload') == 'json_payload') + isJson = request.WhichOneof("payload") == "json_payload" test_message = _create_test_message(request.message_type) if (not isJson) and (test_message is None): raise ProtocolError("Protobuf request doesn't have specific payload type") try: - if request.WhichOneof('payload') == 'protobuf_payload': + if request.WhichOneof("payload") == "protobuf_payload": try: test_message.ParseFromString(request.protobuf_payload) except message.DecodeError as e: response.parse_error = str(e) return response - elif request.WhichOneof('payload') == 'json_payload': + elif request.WhichOneof("payload") == "json_payload": try: - ignore_unknown_fields = \ - request.test_category == \ - conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST - json_format.Parse(request.json_payload, test_message, - ignore_unknown_fields) + ignore_unknown_fields = ( + request.test_category + == conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST + ) + json_format.Parse( + request.json_payload, test_message, ignore_unknown_fields + ) except Exception as e: response.parse_error = str(e) return response - elif request.WhichOneof('payload') == 'text_payload': + elif request.WhichOneof("payload") == "text_payload": try: text_format.Parse(request.text_payload, test_message) except Exception as e: @@ -152,7 +154,8 @@ def do_test(request): elif request.requested_output_format == conformance_pb2.TEXT_FORMAT: response.text_payload = text_format.MessageToString( - test_message, print_unknown_fields=request.print_unknown_fields) + test_message, print_unknown_fields=request.print_unknown_fields + ) except Exception as e: response.runtime_error = str(e) @@ -163,7 +166,7 @@ def do_test(request): def do_test_io(): length_bytes = sys.stdin.buffer.read(4) if len(length_bytes) == 0: - return False # EOF + return False # EOF elif len(length_bytes) != 4: raise IOError("I/O error") @@ -183,15 +186,20 @@ def do_test_io(): sys.stdout.buffer.flush() if verbose: - sys.stderr.write("conformance_python: request=%s, response=%s\n" % ( - request.ShortDebugString().c_str(), - response.ShortDebugString().c_str())) + sys.stderr.write( + "conformance_python: request=%s, response=%s\n" + % ( + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str(), + ) + ) global test_count test_count += 1 return True + while True: if not do_test_io(): sys.stderr.write( diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 10b3103ae1..2b0d728ae0 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -21,6 +21,7 @@ #include "google/protobuf/util/field_comparator.h" #include "google/protobuf/util/message_differencer.h" #include "absl/container/btree_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" @@ -29,6 +30,7 @@ #include "conformance/conformance.pb.h" #include "conformance/conformance.pb.h" #include "google/protobuf/descriptor_legacy.h" +#include "google/protobuf/endian.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" @@ -42,6 +44,15 @@ using std::string; namespace { +static void ReplaceAll(std::string& input, std::string replace_word, + std::string replace_by) { + size_t pos = input.find(replace_word); + while (pos != std::string::npos) { + input.replace(pos, replace_word.length(), replace_by); + pos = input.find(replace_word, pos + replace_by.length()); + } +} + static std::string ToOctString(const std::string& binary_string) { std::string oct_string; for (size_t i = 0; i < binary_string.size(); i++) { @@ -57,6 +68,52 @@ static std::string ToOctString(const std::string& binary_string) { return oct_string; } +// Returns full filename path of written .txt file if successful +static std::string ProduceOctalSerialized(const std::string& request, + uint32_t len) { + char* len_split_bytes = static_cast(static_cast(&len)); + + std::string out; + + std::string hex_repr; + for (int i = 0; i < 4; i++) { + auto conversion = (unsigned int)static_cast(len_split_bytes[i]); + std::string hex = absl::StrFormat("\\x%x", conversion); + absl::StrAppend(&hex_repr, hex); + } + + absl::StrAppend(&out, hex_repr); + + absl::StrAppend(&out, ToOctString(request)); + + return out; +} + +static std::string WriteToFile(const std::string& octal_serialized, + const std::string& output_dir, + const std::string& test_name) { + std::string test_name_txt = test_name; + ReplaceAll(test_name_txt, ".", "_"); + absl::StrAppend(&test_name_txt, ".txt"); + std::string full_filename; + if (!output_dir.empty()) { + full_filename = output_dir; + if (*output_dir.rbegin() != '/') { + full_filename.push_back('/'); + } + absl::StrAppend(&full_filename, test_name_txt); + } + std::ofstream os{std::string(full_filename)}; + if (os) { + os << octal_serialized; + return full_filename; + } else { + ABSL_LOG(INFO) << "Failed to open file for debugging: " << full_filename + << "\n"; + return ""; + } +} + // Removes all newlines. static void Normalize(std::string& input) { input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); @@ -376,7 +433,10 @@ void ConformanceTestSuite::RunValidBinaryInputTest( const std::string& equivalent_wire_format, bool require_same_wire_format) { const ConformanceRequest& request = setting.GetRequest(); ConformanceResponse response; - RunTest(setting.GetTestName(), request, &response); + if (!RunTest(setting.GetTestName(), request, &response)) { + return; + } + VerifyResponse(setting, equivalent_wire_format, response, true, require_same_wire_format); } @@ -451,7 +511,7 @@ void ConformanceTestSuite::VerifyResponse( } } -void ConformanceTestSuite::RunTest(const std::string& test_name, +bool ConformanceTestSuite::RunTest(const std::string& test_name, const ConformanceRequest& request, ConformanceResponse* response) { if (test_names_.insert(test_name).second == false) { @@ -462,7 +522,43 @@ void ConformanceTestSuite::RunTest(const std::string& test_name, std::string serialized_response; request.SerializeToString(&serialized_request); - runner_->RunTest(test_name, serialized_request, &serialized_response); + uint32_t len = internal::little_endian::FromHost( + static_cast(serialized_request.size())); + + if (!debug_) { // Not in debug mode. Continue. + } else if (debug_test_names_->erase(test_name) == 1) { + std::string octal = ProduceOctalSerialized(serialized_request, len); + std::string full_filename = WriteToFile(octal, output_dir_, test_name); + if (!full_filename.empty()) { + absl::StrAppendFormat( + &output_, "Produced octal serialized request file for test %s\n", + test_name); + absl::StrAppendFormat( + &output_, + " To pipe the " + "serialized request directly to " + "the " + "testee run from the root of your workspace:\n printf $(" + "<\"%s\") | " + "./bazel-bin/google/protobuf/conformance/%s\n\n", + full_filename, testee_); + absl::StrAppendFormat( + &output_, + " To inspect the wire format of the serialized request run " + "(Disclaimer: This may not work properly on non-Linux platforms):\n " + " " + "contents=$(<\"%s\"); sub=$(cut -d \\\\ -f 6- <<< " + "$contents) ; printf \"\\\\${sub}\" | protoscope \n\n\n", + full_filename); + } + } else { // Test is not ran, as it was not asked to be debugged. + expected_to_fail_.erase(test_name); + return false; + } + + response->set_protobuf_payload(serialized_request); + + runner_->RunTest(test_name, len, serialized_request, &serialized_response); if (!response->ParseFromString(serialized_response)) { response->Clear(); @@ -475,6 +571,7 @@ void ConformanceTestSuite::RunTest(const std::string& test_name, test_name, TruncateRequest(request).ShortDebugString(), TruncateResponse(*response).ShortDebugString()); } + return true; } std::string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) { @@ -511,7 +608,10 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, unexpected_failing_tests_.clear(); unexpected_succeeding_tests_.clear(); - output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n"; + std::string mode = debug_ ? "DEBUG" : "TEST"; + absl::StrAppendFormat( + &output_, "CONFORMANCE %s BEGIN ====================================\n\n", + mode); failure_list_filename_ = filename; expected_to_fail_.clear(); @@ -604,11 +704,9 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, absl::StrAppendFormat(&output_, "CONFORMANCE SUITE %s: %d successes, %zu skipped, " - "%d expected failures, %zu unexpected failures, %zu " - "unexpected_failure_messages.\n", + "%d expected failures, %zu unexpected failures.\n", ok ? "PASSED" : "FAILED", successes_, skipped_.size(), - expected_failures_, unexpected_failing_tests_.size(), - unexpected_failure_messages_.size()); + expected_failures_, unexpected_failing_tests_.size()); absl::StrAppendFormat(&output_, "\n"); output->assign(output_); diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index 2785138482..d10c9836a9 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -15,6 +15,7 @@ #define CONFORMANCE_CONFORMANCE_TEST_H #include +#include #include #include #include @@ -48,13 +49,14 @@ class ConformanceTestRunner { // Call to run a single conformance test. // + // "len" is the byte length of a serialized conformance.ConformanceRequest. // "input" is a serialized conformance.ConformanceRequest. // "output" should be set to a serialized conformance.ConformanceResponse. // // If there is any error in running the test itself, set "runtime_error" in // the response. - virtual void RunTest(const std::string& test_name, const std::string& input, - std::string* output) = 0; + virtual void RunTest(const std::string& test_name, uint32_t len, + const std::string& input, std::string* output) = 0; }; // Test runner that spawns the process being tested and communicates with it @@ -78,8 +80,8 @@ class ForkPipeRunner : public ConformanceTestRunner { ~ForkPipeRunner() override = default; - void RunTest(const std::string& test_name, const std::string& request, - std::string* response) override; + void RunTest(const std::string& test_name, uint32_t len, + const std::string& request, std::string* response) override; private: void SpawnTestProgram(); @@ -127,11 +129,14 @@ class ForkPipeRunner : public ConformanceTestRunner { class ConformanceTestSuite { public: ConformanceTestSuite() - : verbose_(false), + : testee_(""), + verbose_(false), performance_(false), enforce_recommended_(false), maximum_edition_(Edition::EDITION_PROTO3), - failure_list_flag_name_("--failure_list") {} + failure_list_flag_name_("--failure_list"), + debug_test_names_(nullptr), + debug_(false) {} virtual ~ConformanceTestSuite() = default; void SetPerformance(bool performance) { performance_ = performance; } @@ -159,7 +164,18 @@ class ConformanceTestSuite { } // Sets the path of the output directory. - void SetOutputDir(const char* output_dir) { output_dir_ = output_dir; } + void SetOutputDir(const std::string& output_dir) { output_dir_ = output_dir; } + + // Sets if we are running the test in debug mode. + void SetDebug(bool debug) { debug_ = debug; } + + // Sets the testee name + void SetTestee(const std::string& testee) { testee_ = testee; } + + // Sets the debug test names + void SetDebugTestNames(absl::flat_hash_set& debug_test_names) { + debug_test_names_ = &debug_test_names; + } // Run all the conformance tests against the given test runner. // Test output will be stored in "output". @@ -168,6 +184,9 @@ class ConformanceTestSuite { // failure list. // The filename here is *only* used to create/format useful error messages for // how to update the failure list. We do NOT read this file at all. + + // "debug_test_names" holds the list of test names that the user requested to + // debug. If this is empty, we will run all the tests. bool RunSuite(ConformanceTestRunner* runner, std::string* output, const std::string& filename, conformance::FailureSet* failure_list); @@ -271,7 +290,8 @@ class ConformanceTestSuite { const std::string& equivalent_wire_format, bool require_same_wire_format = false); - void RunTest(const std::string& test_name, + // Returns true if our runner_ ran the test and false if it did not. + bool RunTest(const std::string& test_name, const conformance::ConformanceRequest& request, conformance::ConformanceResponse* response); @@ -280,6 +300,7 @@ class ConformanceTestSuite { virtual void RunSuiteImpl() = 0; ConformanceTestRunner* runner_; + std::string testee_; int successes_; int expected_failures_; bool verbose_; @@ -290,6 +311,8 @@ class ConformanceTestSuite { std::string output_dir_; std::string failure_list_flag_name_; std::string failure_list_filename_; + absl::flat_hash_set* debug_test_names_; + bool debug_; // The set of test names that are expected to fail in this run, but haven't // failed yet. diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index 49aaea374b..3fbd9e2d44 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -47,9 +48,11 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_log.h" #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" #include "conformance/conformance.pb.h" #include "conformance/conformance.pb.h" #include "conformance_test.h" @@ -122,7 +125,7 @@ void UsageError() { fprintf(stderr, " should contain one test name per\n"); fprintf(stderr, - " line. Use '#' for comments.\n"); + " line. Use '#' for comments.\n\n"); fprintf(stderr, " --text_format_failure_list Use to specify list \n"); fprintf(stderr, @@ -133,7 +136,7 @@ void UsageError() { fprintf(stderr, " File should contain one test name \n"); fprintf(stderr, - " per line. Use '#' for comments.\n"); + " per line. Use '#' for comments.\n\n"); fprintf(stderr, " --enforce_recommended Enforce that recommended test\n"); @@ -143,19 +146,30 @@ void UsageError() { " this flag if you want to be\n"); fprintf(stderr, " strictly conforming to protobuf\n"); - fprintf(stderr, " spec.\n"); + fprintf(stderr, " spec.\n\n"); fprintf(stderr, - " --maximum_edition Only run conformance tests up\n"); + " --maximum_edition Only run conformance tests up to\n"); fprintf(stderr, - " to and including the specified\n"); - fprintf(stderr, " edition.\n"); + " and including the specified\n"); + fprintf(stderr, " edition.\n\n"); fprintf(stderr, " --output_dir Directory to write\n" - " output files.\n"); + " output files.\n\n"); + fprintf(stderr, + " --debug ... Debug the \n"); + fprintf(stderr, " specified tests by running\n"); + fprintf(stderr, + " them in isolation and producing\n"); + fprintf(stderr, + " serialized request data for piping\n"); + fprintf(stderr, " directly to the testee.\n\n"); + fprintf(stderr, " --performance Boolean option\n"); + fprintf(stderr, " for enabling run of\n"); + fprintf(stderr, " performance tests.\n"); exit(1); } -void ForkPipeRunner::RunTest(const std::string &test_name, +void ForkPipeRunner::RunTest(const std::string &test_name, uint32_t len, const std::string &request, std::string *response) { if (child_pid_ < 0) { @@ -163,8 +177,6 @@ void ForkPipeRunner::RunTest(const std::string &test_name, } current_test_name_ = test_name; - uint32_t len = - internal::little_endian::FromHost(static_cast(request.size())); CheckedWrite(write_fd_, &len, sizeof(uint32_t)); CheckedWrite(write_fd_, request.c_str(), request.size()); @@ -210,57 +222,97 @@ int ForkPipeRunner::Run(int argc, char *argv[], fprintf(stderr, "No test suites found.\n"); return EXIT_FAILURE; } + + string program; + string testee; + std::vector program_args; + bool performance = false; + bool debug = false; + absl::flat_hash_set debug_test_names; + bool enforce_recommended = false; + Edition maximum_edition = EDITION_UNKNOWN; + std::string output_dir; + bool verbose = false; + + for (int arg = 1; arg < argc; ++arg) { + if (strcmp(argv[arg], "--performance") == 0) { + performance = true; + } else if (strcmp(argv[arg], "--verbose") == 0) { + verbose = true; + } else if (strcmp(argv[arg], "--enforce_recommended") == 0) { + enforce_recommended = true; + } else if (strcmp(argv[arg], "--maximum_edition") == 0) { + if (++arg == argc) UsageError(); + Edition edition = EDITION_UNKNOWN; + if (!Edition_Parse(absl::StrCat("EDITION_", argv[arg]), &edition)) { + fprintf(stderr, "Unknown edition: %s\n", argv[arg]); + UsageError(); + } + maximum_edition = edition; + } else if (strcmp(argv[arg], "--output_dir") == 0) { + if (++arg == argc) UsageError(); + output_dir = argv[arg]; + + } else if (strcmp(argv[arg], "--debug") == 0) { + if (++arg == argc) UsageError(); + for (int debug_arg = arg; debug_arg < argc; ++debug_arg) { + // Stop when we either find another flag or we reach the last arg + // (program arg) + if (argv[debug_arg][0] == '-' || debug_arg == argc - 1) { + arg = debug_arg - 1; + break; + } + debug_test_names.insert(argv[debug_arg]); + } + + } else if (argv[arg][0] == '-') { + bool recognized_flag = false; + for (ConformanceTestSuite *suite : suites) { + if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { + if (++arg == argc) UsageError(); + recognized_flag = true; + } + } + if (!recognized_flag) { + fprintf(stderr, "Unknown option: %s\n", argv[arg]); + UsageError(); + } + } else { + program += argv[arg++]; + while (arg < argc) { + program_args.push_back(argv[arg]); + arg++; + } + } + } + + if (!debug_test_names.empty()) { + debug = true; + } + auto last_slash = program.find_last_of('/'); + if (last_slash != string::npos) { + testee = program.substr(last_slash + 1); + } + bool all_ok = true; for (ConformanceTestSuite *suite : suites) { - string program; - std::vector program_args; string failure_list_filename; conformance::FailureSet failure_list; - - bool performance = false; for (int arg = 1; arg < argc; ++arg) { if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { if (++arg == argc) UsageError(); failure_list_filename = argv[arg]; ParseFailureList(argv[arg], &failure_list); - } else if (strcmp(argv[arg], "--performance") == 0) { - performance = true; - suite->SetPerformance(true); - } else if (strcmp(argv[arg], "--verbose") == 0) { - suite->SetVerbose(true); - } else if (strcmp(argv[arg], "--enforce_recommended") == 0) { - suite->SetEnforceRecommended(true); - } else if (strcmp(argv[arg], "--maximum_edition") == 0) { - if (++arg == argc) UsageError(); - Edition edition = EDITION_UNKNOWN; - if (!Edition_Parse(absl::StrCat("EDITION_", argv[arg]), &edition)) { - fprintf(stderr, "Unknown edition: %s\n", argv[arg]); - UsageError(); - } - suite->SetMaximumEdition(edition); - } else if (strcmp(argv[arg], "--output_dir") == 0) { - if (++arg == argc) UsageError(); - suite->SetOutputDir(argv[arg]); - } else if (argv[arg][0] == '-') { - bool recognized_flag = false; - for (ConformanceTestSuite *suite : suites) { - if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { - if (++arg == argc) UsageError(); - recognized_flag = true; - } - } - if (!recognized_flag) { - fprintf(stderr, "Unknown option: %s\n", argv[arg]); - UsageError(); - } - } else { - program += argv[arg++]; - while (arg < argc) { - program_args.push_back(argv[arg]); - arg++; - } } } + suite->SetPerformance(performance); + suite->SetVerbose(verbose); + suite->SetEnforceRecommended(enforce_recommended); + suite->SetMaximumEdition(maximum_edition); + suite->SetOutputDir(output_dir); + suite->SetDebug(debug); + suite->SetDebugTestNames(debug_test_names); + suite->SetTestee(testee); ForkPipeRunner runner(program, program_args, performance); @@ -270,6 +322,16 @@ int ForkPipeRunner::Run(int argc, char *argv[], fwrite(output.c_str(), 1, output.size(), stderr); } + + if (!debug_test_names.empty()) { + fprintf(stderr, + "These tests were requested to be debugged, but they do " + "not exist. Revise the test names:\n\n"); + for (const string &test_name : debug_test_names) { + fprintf(stderr, " %s\n", test_name.c_str()); + } + fprintf(stderr, "\n\n"); + } return all_ok ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index 2f6a609fa0..508206f5eb 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -175,7 +175,10 @@ void TextFormatConformanceTestSuiteImpl::ExpectParseFailure( setting.ConformanceLevelToString(level), ".", setting.GetSyntaxIdentifier(), ".TextFormatInput.", test_name); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + TestStatus test; test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { From 5854e2b6a823b880f74767debd3d414cf3ab8647 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 29 Jul 2024 02:27:38 -0700 Subject: [PATCH 096/107] Implement std::error::Error for String This enables the use of `proto_str.to_str()?` with anyhow::Result. PiperOrigin-RevId: 657122601 --- rust/string.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/rust/string.rs b/rust/string.rs index 321e035acb..32421e53f3 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -150,11 +150,20 @@ impl<'msg> ViewProxy<'msg> for &'msg [u8] {} /// The bytes were not valid UTF-8. #[derive(Debug, PartialEq)] -pub struct Utf8Error(pub(crate) ()); +pub struct Utf8Error { + pub(crate) inner: std::str::Utf8Error, +} +impl std::fmt::Display for Utf8Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.inner.fmt(f) + } +} + +impl std::error::Error for Utf8Error {} impl From for Utf8Error { - fn from(_: std::str::Utf8Error) -> Utf8Error { - Utf8Error(()) + fn from(inner: std::str::Utf8Error) -> Utf8Error { + Utf8Error { inner } } } @@ -635,7 +644,10 @@ mod tests { b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar", b"\xED\xA0\x80foo\xED\xBF\xBFbar", ] { - assert_eq!(test_proto_str(expect_fail).to_str(), Err(Utf8Error(())), "{expect_fail:?}"); + assert!( + matches!(test_proto_str(expect_fail).to_str(), Err(Utf8Error { inner: _ })), + "{expect_fail:?}" + ); } } From 9f5aed39eace818ca574cde02c222e60e730f6d8 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Mon, 29 Jul 2024 06:44:49 -0700 Subject: [PATCH 097/107] Migrate all of `protos::internal` into `hpb::internal` `hpb.cc` has also been transformed to be 100% hpb. PiperOrigin-RevId: 657183219 --- hpb/extension_lock_test.cc | 2 +- hpb/hpb.cc | 8 +- hpb/hpb.h | 155 +++++++++++++------------- hpb/repeated_field.h | 4 +- hpb_generator/gen_messages.cc | 29 +++-- hpb_generator/tests/test_generated.cc | 6 +- protos/protos.h | 10 ++ 7 files changed, 107 insertions(+), 107 deletions(-) diff --git a/hpb/extension_lock_test.cc b/hpb/extension_lock_test.cc index ef8013bdb8..8e5b46f021 100644 --- a/hpb/extension_lock_test.cc +++ b/hpb/extension_lock_test.cc @@ -54,7 +54,7 @@ void unlock_func(const void* msg) { m[absl::HashOf(msg) & 0x7].unlock(); } return &unlock_func; } -void TestConcurrentExtensionAccess(::protos::ExtensionRegistry registry) { +void TestConcurrentExtensionAccess(::hpb::ExtensionRegistry registry) { ::hpb::internal::upb_extension_locker_global.store(&lock_func, std::memory_order_release); const std::string payload = GenerateTestData(); diff --git a/hpb/hpb.cc b/hpb/hpb.cc index 3437a5fb49..a170dbf6fa 100644 --- a/hpb/hpb.cc +++ b/hpb/hpb.cc @@ -80,15 +80,9 @@ absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc // return absl::Status(absl::StatusCode::kUnknown, "Upb message parse error"); // } // end:github_only -} // namespace hpb -namespace protos { namespace internal { -using ::hpb::internal::upb_extension_locker_global; -using ::hpb::internal::UpbExtensionLocker; -using ::hpb::internal::UpbExtensionUnlocker; - upb_ExtensionRegistry* GetUpbExtensions( const ExtensionRegistry& extension_registry) { return extension_registry.registry_; @@ -188,4 +182,4 @@ absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, } // namespace internal -} // namespace protos +} // namespace hpb diff --git a/hpb/hpb.h b/hpb/hpb.h index 808b9a5bff..ccb649ec64 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -23,6 +23,7 @@ #include "upb/wire/encode.h" namespace hpb { +class ExtensionRegistry; using Arena = ::upb::Arena; template @@ -232,22 +233,6 @@ class ExtensionIdentifier : public ExtensionMiniTableProvider { friend class PrivateAccess; }; -} // namespace internal - -} // namespace hpb - -namespace protos { -using hpb::Arena; -using hpb::ExtensionNotFoundError; -using hpb::MessageAllocationError; -using hpb::MessageDecodeError; -using hpb::MessageEncodeError; -using hpb::Ptr; -using hpb::SourceLocation; -class ExtensionRegistry; - -namespace internal { - template upb_Arena* GetArena(Ptr message) { return static_cast(message->GetInternalArena()); @@ -294,8 +279,45 @@ absl::Status MoveExtension(upb_Message* message, upb_Arena* message_arena, absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, const upb_MiniTableExtension* ext, const upb_Message* extension); + } // namespace internal +class ExtensionRegistry { + public: + ExtensionRegistry( + const std::vector& + extensions, + const upb::Arena& arena) + : registry_(upb_ExtensionRegistry_New(arena.ptr())) { + if (registry_) { + for (const auto& ext_provider : extensions) { + const auto* ext = ext_provider->mini_table_ext(); + bool success = upb_ExtensionRegistry_AddArray(registry_, &ext, 1); + if (!success) { + registry_ = nullptr; + break; + } + } + } + } + + private: + friend upb_ExtensionRegistry* ::hpb::internal::GetUpbExtensions( + const ExtensionRegistry& extension_registry); + upb_ExtensionRegistry* registry_; +}; + +} // namespace hpb + +namespace protos { +using hpb::Arena; +using hpb::ExtensionNotFoundError; +using hpb::MessageAllocationError; +using hpb::MessageDecodeError; +using hpb::MessageEncodeError; +using hpb::Ptr; +using hpb::SourceLocation; + template typename T::Proxy CreateMessage(::hpb::Arena& arena) { return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), @@ -305,7 +327,7 @@ typename T::Proxy CreateMessage(::hpb::Arena& arena) { template void DeepCopy(Ptr source_message, Ptr target_message) { static_assert(!std::is_const_v); - ::protos::internal::DeepCopy( + ::hpb::internal::DeepCopy( hpb::internal::GetInternalMsg(target_message), hpb::internal::GetInternalMsg(source_message), T::minitable(), static_cast(target_message->GetInternalArena())); @@ -313,9 +335,9 @@ void DeepCopy(Ptr source_message, Ptr target_message) { template typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { - return hpb::internal::PrivateAccess::Proxy( - ::protos::internal::DeepClone(hpb::internal::GetInternalMsg(message), - T::minitable(), arena), + return ::hpb::internal::PrivateAccess::Proxy( + ::hpb::internal::DeepClone(::hpb::internal::GetInternalMsg(message), + T::minitable(), arena), arena); } @@ -340,41 +362,16 @@ void DeepCopy(const T* source_message, T* target_message) { template void ClearMessage(hpb::internal::PtrOrRaw message) { auto ptr = Ptr(message); - auto minitable = internal::GetMiniTable(ptr); + auto minitable = hpb::internal::GetMiniTable(ptr); upb_Message_Clear(hpb::internal::GetInternalMsg(ptr), minitable); } -class ExtensionRegistry { - public: - ExtensionRegistry( - const std::vector& - extensions, - const upb::Arena& arena) - : registry_(upb_ExtensionRegistry_New(arena.ptr())) { - if (registry_) { - for (const auto& ext_provider : extensions) { - const auto* ext = ext_provider->mini_table_ext(); - bool success = upb_ExtensionRegistry_AddArray(registry_, &ext, 1); - if (!success) { - registry_ = nullptr; - break; - } - } - } - } - - private: - friend upb_ExtensionRegistry* ::protos::internal::GetUpbExtensions( - const ExtensionRegistry& extension_registry); - upb_ExtensionRegistry* registry_; -}; - template > ABSL_MUST_USE_RESULT bool HasExtension( Ptr message, const ::hpb::internal::ExtensionIdentifier& id) { - return ::protos::internal::HasExtensionOrUnknown( + return ::hpb::internal::HasExtensionOrUnknown( ::hpb::internal::GetInternalMsg(message), id.mini_table_ext()); } @@ -413,9 +410,9 @@ absl::Status SetExtension( const Extension& value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); - return ::protos::internal::SetExtension( - hpb::internal::GetInternalMsg(message), message_arena, - id.mini_table_ext(), hpb::internal::GetInternalMsg(&value)); + return ::hpb::internal::SetExtension(hpb::internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + hpb::internal::GetInternalMsg(&value)); } template value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); - return ::protos::internal::SetExtension( - hpb::internal::GetInternalMsg(message), message_arena, - id.mini_table_ext(), hpb::internal::GetInternalMsg(value)); + return ::hpb::internal::SetExtension(hpb::internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + hpb::internal::GetInternalMsg(value)); } template ); auto* message_arena = static_cast(message->GetInternalArena()); auto* extension_arena = static_cast(ext.GetInternalArena()); - return ::protos::internal::MoveExtension( - hpb::internal::GetInternalMsg(message), message_arena, - id.mini_table_ext(), hpb::internal::GetInternalMsg(&ext), - extension_arena); + return ::hpb::internal::MoveExtension(hpb::internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + hpb::internal::GetInternalMsg(&ext), + extension_arena); } template > GetExtension( const ::hpb::internal::ExtensionIdentifier& id) { // TODO: Fix const correctness issues. upb_MessageValue value; - const bool ok = ::protos::internal::GetOrPromoteExtension( + const bool ok = ::hpb::internal::GetOrPromoteExtension( const_cast(::hpb::internal::GetInternalMsg(message)), - id.mini_table_ext(), ::protos::internal::GetArena(message), &value); + id.mini_table_ext(), ::hpb::internal::GetArena(message), &value); if (!ok) { return ExtensionNotFoundError( upb_MiniTableExtension_Number(id.mini_table_ext())); } return Ptr(::hpb::internal::CreateMessage( - (upb_Message*)value.msg_val, ::protos::internal::GetArena(message))); + (upb_Message*)value.msg_val, ::hpb::internal::GetArena(message))); } template ABSL_MUST_USE_RESULT bool Parse(Ptr message, absl::string_view bytes) { static_assert(!std::is_const_v); upb_Message_Clear(::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message)); + ::hpb::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), ::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), + ::hpb::internal::GetMiniTable(message), /* extreg= */ nullptr, /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; } @@ -516,23 +513,23 @@ ABSL_MUST_USE_RESULT bool Parse(Ptr message, absl::string_view bytes) { template ABSL_MUST_USE_RESULT bool Parse( Ptr message, absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry) { + const ::hpb::ExtensionRegistry& extension_registry) { static_assert(!std::is_const_v); upb_Message_Clear(::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message)); + ::hpb::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), ::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), + ::hpb::internal::GetMiniTable(message), /* extreg= */ - ::protos::internal::GetUpbExtensions(extension_registry), + ::hpb::internal::GetUpbExtensions(extension_registry), /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; } template ABSL_MUST_USE_RESULT bool Parse( T* message, absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry) { + const ::hpb::ExtensionRegistry& extension_registry) { static_assert(!std::is_const_v); return Parse(Ptr(message, bytes, extension_registry)); } @@ -541,11 +538,11 @@ template ABSL_MUST_USE_RESULT bool Parse(T* message, absl::string_view bytes) { static_assert(!std::is_const_v); upb_Message_Clear(::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message)); + ::hpb::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), ::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), + ::hpb::internal::GetMiniTable(message), /* extreg= */ nullptr, /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; } @@ -556,7 +553,7 @@ absl::StatusOr Parse(absl::string_view bytes, int options = 0) { auto* arena = static_cast(message.GetInternalArena()); upb_DecodeStatus status = upb_Decode(bytes.data(), bytes.size(), message.msg(), - ::protos::internal::GetMiniTable(&message), + ::hpb::internal::GetMiniTable(&message), /* extreg= */ nullptr, /* options= */ 0, arena); if (status == kUpb_DecodeStatus_Ok) { return message; @@ -566,14 +563,14 @@ absl::StatusOr Parse(absl::string_view bytes, int options = 0) { template absl::StatusOr Parse(absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry, + const ::hpb::ExtensionRegistry& extension_registry, int options = 0) { T message; auto* arena = static_cast(message.GetInternalArena()); upb_DecodeStatus status = upb_Decode(bytes.data(), bytes.size(), message.msg(), - ::protos::internal::GetMiniTable(&message), - ::protos::internal::GetUpbExtensions(extension_registry), + ::hpb::internal::GetMiniTable(&message), + ::hpb::internal::GetUpbExtensions(extension_registry), /* options= */ 0, arena); if (status == kUpb_DecodeStatus_Ok) { return message; @@ -584,17 +581,17 @@ absl::StatusOr Parse(absl::string_view bytes, template absl::StatusOr Serialize(const T* message, upb::Arena& arena, int options = 0) { - return ::protos::internal::Serialize( - ::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), arena.ptr(), options); + return ::hpb::internal::Serialize(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), + arena.ptr(), options); } template absl::StatusOr Serialize(Ptr message, upb::Arena& arena, int options = 0) { - return ::protos::internal::Serialize( - ::hpb::internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), arena.ptr(), options); + return ::hpb::internal::Serialize(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), + arena.ptr(), options); } template diff --git a/hpb/repeated_field.h b/hpb/repeated_field.h index 04bdee401a..7810a043f2 100644 --- a/hpb/repeated_field.h +++ b/hpb/repeated_field.h @@ -120,7 +120,7 @@ class RepeatedFieldProxy upb_MessageValue message_value; message_value.msg_val = upb_Message_DeepClone( ::hpb::internal::PrivateAccess::GetInternalMsg(&t), - ::protos::internal::GetMiniTable(&t), this->arena_); + ::hpb::internal::GetMiniTable(&t), this->arena_); upb_Array_Append(this->arr_, message_value, this->arena_); } @@ -131,7 +131,7 @@ class RepeatedFieldProxy upb_MessageValue message_value; message_value.msg_val = ::hpb::internal::PrivateAccess::GetInternalMsg(&msg); - upb_Arena_Fuse(::protos::internal::GetArena(&msg), this->arena_); + upb_Arena_Fuse(::hpb::internal::GetArena(&msg), this->arena_); upb_Array_Append(this->arr_, message_value, this->arena_); T moved_msg = std::move(msg); } diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index a43dc1df96..7de1b213eb 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -245,10 +245,9 @@ void WriteModelPublicDeclaration( int options)); friend absl::StatusOr<$2>(::protos::Parse<$2>( absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry, - int options)); - friend upb_Arena* ::protos::internal::GetArena<$0>($0* message); - friend upb_Arena* ::protos::internal::GetArena<$0>(::hpb::Ptr<$0> message); + const ::hpb::ExtensionRegistry& extension_registry, int options)); + friend upb_Arena* ::hpb::internal::GetArena<$0>($0* message); + friend upb_Arena* ::hpb::internal::GetArena<$0>(::hpb::Ptr<$0> message); friend $0(::hpb::internal::MoveMessage<$0>(upb_Message* msg, upb_Arena* arena)); )cc", ClassName(descriptor), MessageName(descriptor), @@ -303,12 +302,12 @@ void WriteModelProxyDeclaration(const protobuf::Descriptor* descriptor, friend class ::hpb::Ptr<$0>; friend class ::hpb::Ptr; static const upb_MiniTable* minitable() { return $0::minitable(); } - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0Proxy>( + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0Proxy>( const $0Proxy* message); - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0Proxy>( + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0Proxy>( ::hpb::Ptr<$0Proxy> message); - friend upb_Arena* ::protos::internal::GetArena<$2>($2* message); - friend upb_Arena* ::protos::internal::GetArena<$2>(::hpb::Ptr<$2> message); + friend upb_Arena* ::hpb::internal::GetArena<$2>($2* message); + friend upb_Arena* ::hpb::internal::GetArena<$2>(::hpb::Ptr<$2> message); static void Rebind($0Proxy& lhs, const $0Proxy& rhs) { lhs.msg_ = rhs.msg_; lhs.arena_ = rhs.arena_; @@ -329,7 +328,7 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, public: $0CProxy() = delete; $0CProxy(const $0* m) - : internal::$0Access(m->msg_, ::protos::internal::GetArena(m)) {} + : internal::$0Access(m->msg_, ::hpb::internal::GetArena(m)) {} $0CProxy($0Proxy m); using $0Access::GetInternalArena; )cc", @@ -352,9 +351,9 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, friend class ::hpb::Ptr<$0>; friend class ::hpb::Ptr; static const upb_MiniTable* minitable() { return $0::minitable(); } - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0CProxy>( + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0CProxy>( const $0CProxy* message); - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0CProxy>( + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0CProxy>( ::hpb::Ptr<$0CProxy> message); static void Rebind($0CProxy& lhs, const $0CProxy& rhs) { @@ -388,11 +387,11 @@ void WriteMessageImplementation( } $0::$0(const $0& from) : $0Access() { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); + msg_ = ($1*)::hpb::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); } $0::$0(const CProxy& from) : $0Access() { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone( + msg_ = ($1*)::hpb::internal::DeepClone( ::hpb::internal::GetInternalMsg(&from), &$2, arena_); } $0::$0(const Proxy& from) : $0(static_cast(from)) {} @@ -402,12 +401,12 @@ void WriteMessageImplementation( } $0& $0::operator=(const $3& from) { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); + msg_ = ($1*)::hpb::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); return *this; } $0& $0::operator=(const CProxy& from) { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone( + msg_ = ($1*)::hpb::internal::DeepClone( ::hpb::internal::GetInternalMsg(&from), &$2, arena_); return *this; } diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index eabcb075b5..3cb51a1c2f 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -797,7 +797,7 @@ TEST(CppGeneratedCode, SetExtensionFusingFailureShouldCopy) { ThemeExtension extension1; extension1.set_ext_name("Hello World"); ASSERT_FALSE( - upb_Arena_Fuse(arena.ptr(), ::protos::internal::GetArena(&extension1))); + upb_Arena_Fuse(arena.ptr(), ::hpb::internal::GetArena(&extension1))); EXPECT_FALSE(::protos::HasExtension(model, theme)); auto status = ::protos::SetExtension(model, theme, std::move(extension1)); EXPECT_TRUE(status.ok()); @@ -965,7 +965,7 @@ TEST(CppGeneratedCode, ParseWithExtensionRegistry) { ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); - ::protos::ExtensionRegistry extensions( + ::hpb::ExtensionRegistry extensions( {&theme, &other_ext, &ThemeExtension::theme_extension}, arena); TestModel parsed_model = ::protos::Parse(bytes.value(), extensions).value(); @@ -1230,7 +1230,7 @@ TEST(CppGeneratedCode, HasExtensionAndRegistry) { std::string data = std::string(::protos::Serialize(&source, arena).value()); // Test with ExtensionRegistry - ::protos::ExtensionRegistry extensions({&theme}, arena); + ::hpb::ExtensionRegistry extensions({&theme}, arena); TestModel parsed_model = ::protos::Parse(data, extensions).value(); EXPECT_TRUE(::protos::HasExtension(&parsed_model, theme)); } diff --git a/protos/protos.h b/protos/protos.h index aee9659383..250152d41c 100644 --- a/protos/protos.h +++ b/protos/protos.h @@ -12,9 +12,19 @@ namespace protos { namespace internal { using hpb::internal::CreateMessage; using hpb::internal::CreateMessageProxy; +using hpb::internal::DeepClone; +using hpb::internal::DeepCopy; using hpb::internal::ExtensionIdentifier; +using hpb::internal::GetArena; using hpb::internal::GetInternalMsg; +using hpb::internal::GetMiniTable; +using hpb::internal::GetOrPromoteExtension; +using hpb::internal::GetUpbExtensions; +using hpb::internal::HasExtensionOrUnknown; +using hpb::internal::MoveExtension; using hpb::internal::PrivateAccess; +using hpb::internal::Serialize; +using hpb::internal::SetExtension; } // namespace internal } // namespace protos #endif From 504a5d03112431f15d822ae3ddb1e2e677da5fb1 Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Mon, 29 Jul 2024 08:50:52 -0700 Subject: [PATCH 098/107] Add pkg build rules to create a zip of the protobuf rust crate PiperOrigin-RevId: 657215370 --- rust/BUILD | 60 +++++++++++++++++++++++++++++++++++++++++++ rust/cargo/Cargo.toml | 12 +++++++-- rust/cargo/build.rs | 19 +++++++++----- rust/upb/BUILD | 17 ++++++++++++ upb/BUILD | 1 + upb/cmake/BUILD.bazel | 24 +++++++++++++++++ 6 files changed, 124 insertions(+), 9 deletions(-) diff --git a/rust/BUILD b/rust/BUILD index d403bb3d5c..43b6e6a49d 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -1,6 +1,12 @@ # Protobuf Rust runtime packages. load("@bazel_skylib//rules:common_settings.bzl", "string_flag") + +# begin:github_only +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") +load("@rules_pkg//pkg:zip.bzl", "pkg_zip") +# end:github_only + load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") @@ -203,3 +209,57 @@ config_setting( ":rust_proto_library_kernel": "upb", }, ) + +# begin:github_only +# pkg_files( +# name = "rust_protobuf_src", +# srcs = glob( +# [ +# "*", +# ], +# ), +# strip_prefix = strip_prefix.from_root("rust"), +# visibility = ["//:__pkg__"], +# ) + +# pkg_files( +# name = "crate_root_files", +# srcs = glob(["cargo/**/*"]), +# strip_prefix = strip_prefix.from_root("rust/cargo"), +# visibility = ["//:__pkg__"], +# ) + +# pkg_filegroup( +# name = "rust_protobuf_src_dir", +# srcs = [ +# ":rust_protobuf_src", +# "//rust/upb:rust_protobuf_upb_src", +# ], +# prefix = "src", +# ) + +# pkg_files( +# name = "amalgamated_upb", +# srcs = ["//upb:gen_amalgamation"], +# strip_prefix = strip_prefix.from_root(""), +# ) + +# pkg_filegroup( +# name = "rust_protobuf_libupb_src", +# srcs = [ +# ":amalgamated_upb", +# "//upb/cmake:upb_cmake_dist", +# ], +# prefix = "libupb", +# ) + +# pkg_zip( +# name = "rust_crate", +# srcs = [ +# ":crate_root_files", +# ":rust_protobuf_libupb_src", +# ":rust_protobuf_src_dir", +# "//:LICENSE", +# ], +# ) +# end:github_only diff --git a/rust/cargo/Cargo.toml b/rust/cargo/Cargo.toml index 2d8e44bb9d..7a389e764e 100644 --- a/rust/cargo/Cargo.toml +++ b/rust/cargo/Cargo.toml @@ -1,8 +1,16 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2024 Google LLC. All rights reserved. + +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + [package] name = "protobuf" version = "4.27.3-beta.0" edition = "2021" -links = "upb" +links = "libupb" +license = "BSD-3-Clause" [lib] path = "src/shared.rs" @@ -14,7 +22,7 @@ paste = "1.0.15" googletest = {git = "https://github.com/google/googletest-rust.git" } [build-dependencies] -cmake = "0.1.50" +cc = "1.1.6" [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bzl)', 'cfg(cpp_kernel)', 'cfg(upb_kernel)'] } \ No newline at end of file diff --git a/rust/cargo/build.rs b/rust/cargo/build.rs index e400e15471..c69ef7c393 100644 --- a/rust/cargo/build.rs +++ b/rust/cargo/build.rs @@ -1,16 +1,21 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. + +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + fn main() { cc::Build::new() .flag("-std=c99") // TODO: Come up with a way to enable lto // .flag("-flto=thin") .warnings(false) - .include("src") - .file("src/upb.c") - .file("src/utf8_range.c") - .file("src/upb/upb_api.c") + .include("libupb") + .file("libupb/upb/upb.c") + .file("libupb/third_party/utf8_range/utf8_range.c") .define("UPB_BUILD_API", Some("1")) - .define("PROTOBUF_CARGO", Some("1")) - .compile("upb"); - let path = std::path::Path::new("src"); + .compile("libupb"); + let path = std::path::Path::new("libupb"); println!("cargo:include={}", path.canonicalize().unwrap().display()) } diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 3cd2c5e84b..4ac6a3f4e4 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -4,6 +4,10 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +# begin:github_only +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") +# end:github_only + load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") # begin:google_only @@ -53,3 +57,16 @@ cc_library( "//upb/text:debug", ], ) + +# begin:github_only +pkg_files( + name = "rust_protobuf_upb_src", + srcs = glob( + [ + "*", + ], + ), + strip_prefix = strip_prefix.from_root("rust"), + visibility = ["//rust:__pkg__"], +) +# end:github_only diff --git a/upb/BUILD b/upb/BUILD index 42797dc2ec..b46ee4b477 100644 --- a/upb/BUILD +++ b/upb/BUILD @@ -288,6 +288,7 @@ upb_amalgamation( "//upb/reflection:internal", ], strip_import_prefix = ["src"], + visibility = ["//rust:__pkg__"], ) cc_library( diff --git a/upb/cmake/BUILD.bazel b/upb/cmake/BUILD.bazel index e17ce8450a..fdf0f23226 100644 --- a/upb/cmake/BUILD.bazel +++ b/upb/cmake/BUILD.bazel @@ -5,6 +5,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load( "//upb/bazel:build_defs.bzl", "make_shell_script", @@ -106,3 +107,26 @@ sh_test( }), deps = ["@bazel_tools//tools/bash/runfiles"], ) + +pkg_files( + name = "upb_cmake_dist", + srcs = [ + ":copy_protos", + ":gen_cmakelists", + "//third_party/utf8_range:utf8_range_srcs", + "//upb:source_files", + "//upb/base:source_files", + "//upb/hash:source_files", + "//upb/lex:source_files", + "//upb/mem:source_files", + "//upb/message:source_files", + "//upb/mini_descriptor:source_files", + "//upb/mini_table:source_files", + "//upb/port:source_files", + "//upb/reflection:source_files", + "//upb/text:source_files", + "//upb/wire:source_files", + ], + strip_prefix = strip_prefix.from_root(""), + visibility = ["//rust:__pkg__"], +) From 1f9206c7036f2876c609c700bc46da2cca907b38 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Mon, 29 Jul 2024 09:36:22 -0700 Subject: [PATCH 099/107] Internal Change PiperOrigin-RevId: 657229104 --- hpb/BUILD | 2 +- hpb_generator/BUILD | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hpb/BUILD b/hpb/BUILD index 3f39415bc0..69587c7fdd 100644 --- a/hpb/BUILD +++ b/hpb/BUILD @@ -15,7 +15,7 @@ load( ) # begin:google_only -# package(default_applicable_licenses = ["//upb:license"]) +# package(default_applicable_licenses = ["//src/google/protobuf:license"]) # end:google_only licenses(["notice"]) diff --git a/hpb_generator/BUILD b/hpb_generator/BUILD index 7ca5c590b9..fba7127106 100644 --- a/hpb_generator/BUILD +++ b/hpb_generator/BUILD @@ -11,7 +11,7 @@ load( ) # begin:google_only -# package(default_applicable_licenses = ["//upb:license"]) +# package(default_applicable_licenses = ["//src/google/protobuf:license"]) # end:google_only licenses(["notice"]) From 51763f3e4085443160c713ba26bf104c58a55c36 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 29 Jul 2024 10:58:40 -0700 Subject: [PATCH 100/107] Automated rollback of commit 74f6da4d26173088de178365678891105393799e. PiperOrigin-RevId: 657257908 --- hpb/hpb.h | 9 ++--- hpb_generator/gen_extensions.cc | 50 ++++++++++++++++++-------- hpb_generator/gen_extensions.h | 8 ++++- hpb_generator/gen_messages.cc | 21 ++++++++++- hpb_generator/protoc-gen-upb-protos.cc | 5 +-- hpb_generator/tests/test_generated.cc | 2 -- 6 files changed, 71 insertions(+), 24 deletions(-) diff --git a/hpb/hpb.h b/hpb/hpb.h index ccb649ec64..b18569cc96 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -224,12 +224,13 @@ class ExtensionIdentifier : public ExtensionMiniTableProvider { using Extendee = ExtendeeType; constexpr explicit ExtensionIdentifier( - uint32_t number, const upb_MiniTableExtension* mini_table_ext) - : ExtensionMiniTableProvider(mini_table_ext), number_(number) {} + const upb_MiniTableExtension* mini_table_ext) + : ExtensionMiniTableProvider(mini_table_ext) {} private: - uint32_t number_; - constexpr uint32_t number() const { return number_; } + constexpr uint32_t number() const { + return upb_MiniTableExtension_Number(mini_table_ext()); + } friend class PrivateAccess; }; diff --git a/hpb_generator/gen_extensions.cc b/hpb_generator/gen_extensions.cc index f5d44ff3d5..00a8dcd496 100644 --- a/hpb_generator/gen_extensions.cc +++ b/hpb_generator/gen_extensions.cc @@ -7,14 +7,9 @@ #include "google/protobuf/compiler/hpb/gen_extensions.h" -#include -#include -#include - #include "absl/strings/str_cat.h" +#include "google/protobuf/compiler/hpb/gen_utils.h" #include "google/protobuf/compiler/hpb/names.h" -#include "google/protobuf/compiler/hpb/output.h" -#include "google/protobuf/descriptor.h" namespace google::protobuf::hpb_generator { @@ -43,19 +38,15 @@ void WriteExtensionIdentifierHeader(const protobuf::FieldDescriptor* ext, if (ext->extension_scope()) { output( R"cc( - static constexpr ::hpb::internal::ExtensionIdentifier<$0, $3> $2{$4, - &$1}; + static const ::hpb::internal::ExtensionIdentifier<$0, $1> $2; )cc", - ContainingTypeName(ext), mini_table_name, ext->name(), - CppTypeParameterName(ext), ext->number()); + ContainingTypeName(ext), CppTypeParameterName(ext), ext->name()); } else { output( R"cc( - inline constexpr ::hpb::internal::ExtensionIdentifier<$0, $3> $2{$4, - &$1}; + extern const ::hpb::internal::ExtensionIdentifier<$0, $1> $2; )cc", - ContainingTypeName(ext), mini_table_name, ext->name(), - CppTypeParameterName(ext), ext->number()); + ContainingTypeName(ext), CppTypeParameterName(ext), ext->name()); } } @@ -69,5 +60,36 @@ void WriteExtensionIdentifiersHeader( } } +void WriteExtensionIdentifier(const protobuf::FieldDescriptor* ext, + Output& output) { + std::string mini_table_name = + absl::StrCat(ExtensionIdentifierBase(ext), "_", ext->name(), "_ext"); + if (ext->extension_scope()) { + output( + R"cc( + const ::protos::internal::ExtensionIdentifier<$0, $3> $4::$2(&$1); + )cc", + ContainingTypeName(ext), mini_table_name, ext->name(), + CppTypeParameterName(ext), ClassName(ext->extension_scope())); + } else { + output( + R"cc( + const ::protos::internal::ExtensionIdentifier<$0, $3> $2(&$1); + )cc", + ContainingTypeName(ext), mini_table_name, ext->name(), + CppTypeParameterName(ext)); + } +} + +void WriteExtensionIdentifiers( + const std::vector& extensions, + Output& output) { + for (const auto* ext : extensions) { + if (!ext->extension_scope()) { + WriteExtensionIdentifier(ext, output); + } + } +} + } // namespace protobuf } // namespace google::hpb_generator diff --git a/hpb_generator/gen_extensions.h b/hpb_generator/gen_extensions.h index 1d2e932c43..e48f657e8d 100644 --- a/hpb_generator/gen_extensions.h +++ b/hpb_generator/gen_extensions.h @@ -8,8 +8,8 @@ #ifndef PROTOBUF_COMPILER_HBP_GEN_EXTENSIONS_H_ #define PROTOBUF_COMPILER_HBP_GEN_EXTENSIONS_H_ -#include "google/protobuf/compiler/hpb/output.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/compiler/hpb/output.h" namespace google::protobuf::hpb_generator { @@ -20,6 +20,12 @@ void WriteExtensionIdentifiersHeader( Output& output); void WriteExtensionIdentifierHeader(const protobuf::FieldDescriptor* ext, Output& output); +void WriteExtensionIdentifiers( + const std::vector& extensions, + Output& output); +void WriteExtensionIdentifier(const protobuf::FieldDescriptor* ext, + Output& output); + } // namespace protobuf } // namespace google::hpb_generator diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index 7de1b213eb..2ea07e7701 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -15,14 +15,15 @@ #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "google/protobuf/descriptor.h" #include "google/protobuf/compiler/hpb/gen_accessors.h" #include "google/protobuf/compiler/hpb/gen_enums.h" #include "google/protobuf/compiler/hpb/gen_extensions.h" #include "google/protobuf/compiler/hpb/gen_utils.h" #include "google/protobuf/compiler/hpb/names.h" #include "google/protobuf/compiler/hpb/output.h" -#include "google/protobuf/descriptor.h" #include "upb_generator/common.h" +#include "upb_generator/file_layout.h" namespace google::protobuf::hpb_generator { @@ -47,6 +48,10 @@ void WriteInternalForwardDeclarationsInHeader( const protobuf::Descriptor* message, Output& output); void WriteDefaultInstanceHeader(const protobuf::Descriptor* message, Output& output); +void WriteExtensionIdentifiersImplementation( + const protobuf::Descriptor* message, + const std::vector& file_exts, + Output& output); void WriteUsingEnumsInHeader( const protobuf::Descriptor* message, const std::vector& file_enums, @@ -451,6 +456,8 @@ void WriteMessageImplementation( } )cc", ClassName(descriptor)); + + WriteExtensionIdentifiersImplementation(descriptor, file_exts, output); } } @@ -478,6 +485,18 @@ void WriteExtensionIdentifiersInClassHeader( } } +void WriteExtensionIdentifiersImplementation( + const protobuf::Descriptor* message, + const std::vector& file_exts, + Output& output) { + for (auto* ext : file_exts) { + if (ext->extension_scope() && + ext->extension_scope()->full_name() == message->full_name()) { + WriteExtensionIdentifier(ext, output); + } + } +} + void WriteUsingEnumsInHeader( const protobuf::Descriptor* message, const std::vector& file_enums, diff --git a/hpb_generator/protoc-gen-upb-protos.cc b/hpb_generator/protoc-gen-upb-protos.cc index 5bf9b8f381..d946fb260a 100644 --- a/hpb_generator/protoc-gen-upb-protos.cc +++ b/hpb_generator/protoc-gen-upb-protos.cc @@ -13,14 +13,14 @@ #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/compiler/plugin.h" +#include "google/protobuf/descriptor.h" #include "google/protobuf/compiler/hpb/gen_enums.h" #include "google/protobuf/compiler/hpb/gen_extensions.h" #include "google/protobuf/compiler/hpb/gen_messages.h" #include "google/protobuf/compiler/hpb/gen_utils.h" #include "google/protobuf/compiler/hpb/names.h" #include "google/protobuf/compiler/hpb/output.h" -#include "google/protobuf/compiler/plugin.h" -#include "google/protobuf/descriptor.h" namespace google::protobuf::hpb_generator { namespace { @@ -211,6 +211,7 @@ void WriteSource(const protobuf::FileDescriptor* file, Output& output, WriteMessageImplementations(file, output); const std::vector this_file_exts = SortedExtensions(file); + WriteExtensionIdentifiers(this_file_exts, output); WriteEndNamespace(file, output); output("#include \"upb/port/undef.inc\"\n\n"); diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index 3cb51a1c2f..b09787590b 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -1241,8 +1241,6 @@ TEST(CppGeneratedCode, FieldNumberConstants) { } TEST(CppGeneratedCode, ExtensionFieldNumberConstant) { - static_assert(::protos::ExtensionNumber(ThemeExtension::theme_extension) == - 12003); EXPECT_EQ(12003, ::protos::ExtensionNumber(ThemeExtension::theme_extension)); } From 7ad56e7cbb60065bc58ff9523fe88422e3591f60 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 29 Jul 2024 11:13:57 -0700 Subject: [PATCH 101/107] Introduce an upb::AssociatedMiniTable trait in Rust. PiperOrigin-RevId: 657263802 --- rust/upb/BUILD | 1 + rust/upb/associated_mini_table.rs | 30 ++++++++++ rust/upb/lib.rs | 3 + .../compiler/rust/accessors/repeated_field.cc | 2 +- .../protobuf/compiler/rust/generator.cc | 10 ++++ src/google/protobuf/compiler/rust/message.cc | 60 +++++++++---------- 6 files changed, 75 insertions(+), 31 deletions(-) create mode 100644 rust/upb/associated_mini_table.rs diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 4ac6a3f4e4..8fd5505a04 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -19,6 +19,7 @@ rust_library( srcs = [ "arena.rs", "array.rs", + "associated_mini_table.rs", "ctype.rs", "extension_registry.rs", "lib.rs", diff --git a/rust/upb/associated_mini_table.rs b/rust/upb/associated_mini_table.rs new file mode 100644 index 0000000000..382d9a63ac --- /dev/null +++ b/rust/upb/associated_mini_table.rs @@ -0,0 +1,30 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +use super::upb_MiniTable; + +/// A trait for types which have a constant associated MiniTable (e.g. +/// generated messages, and their mut and view proxy types). +/// +/// upb_Message in C is effectively a DST type, where instances are created with +/// a MiniTable (and have a size dependent on the given MiniTable). Many upb +/// operations on the upb_Message* will require the same upb_MiniTable* to be +/// passed in as a parameter, which is referred to as 'the associated MiniTable +/// for the upb_Message instance' in safety comments. +/// +/// This trait is a way to statically associate a MiniTable with Rust types +/// which hold upb_Message* to simplify ensuring the upb C invariants +/// are maintained. +/// +/// SAFETY: +/// - The MiniTable pointer must be from Protobuf code generation and follow the +/// corresponding invariants associated with upb's C API (the pointer should +/// always have the same non-null value, the underlying pointee should never +/// be modified and should have 'static lifetime). +pub unsafe trait AssociatedMiniTable { + const MINI_TABLE: *const upb_MiniTable; +} diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 84b09fc5de..125a0f277a 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -18,6 +18,9 @@ pub use array::{ upb_Array_Size, RawArray, }; +mod associated_mini_table; +pub use associated_mini_table::AssociatedMiniTable; + mod ctype; pub use ctype::CType; diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index 966f835608..a8f6e2b13b 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -114,7 +114,7 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, pub fn set_$raw_field_name$(&mut self, src: impl $pb$::IntoProxied<$pb$::Repeated<$RsType$>>) { let minitable_field = unsafe { $pbr$::upb_MiniTable_FindFieldByNumber( - Self::raw_minitable($pbi$::Private), + ::MINI_TABLE, $field_number$ ) }; diff --git a/src/google/protobuf/compiler/rust/generator.cc b/src/google/protobuf/compiler/rust/generator.cc index dc49e325b0..52a120f409 100644 --- a/src/google/protobuf/compiler/rust/generator.cc +++ b/src/google/protobuf/compiler/rust/generator.cc @@ -177,6 +177,16 @@ bool RustGenerator::Generate(const FileDescriptor* file, {"Phantom", "::__std::marker::PhantomData"}, }); + // On upb we need to enable this feature to be able to have const pointers to + // upb_MiniTables while using the upb C minitable codegen. This + // feature appears that it'll imminently be considered stabilized + // (https://github.com/rust-lang/rust/issues/128183), and if we emit the + // MiniTables as const directly in .rs rather than using the upb minitable .h + // this could be avoided regardless. + if (ctx.is_upb() && file == &rust_generator_context.primary_file()) { + ctx.Emit("#![feature(const_refs_to_static)]\n"); + } + ctx.Emit({{"kernel", KernelRsName(ctx.opts().kernel)}}, R"rs( extern crate protobuf_$kernel$ as __pb; extern crate std as __std; diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index dd7369f299..6b652f4af9 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -76,11 +76,10 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( - // SAFETY: $minitable$ is a static of a const object. - let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; - // SAFETY: $minitable$ is the one associated with raw_msg(). + // SAFETY: `MINI_TABLE` is the one associated with `self.raw_msg()`. let encoded = unsafe { - $pbr$::wire::encode(self.raw_msg(), mini_table) + $pbr$::wire::encode(self.raw_msg(), + ::MINI_TABLE) }; //~ TODO: This discards the info we have about the reason //~ of the failure, we should try to keep it instead. @@ -139,8 +138,6 @@ void MessageClearAndParse(Context& ctx, const Descriptor& msg) { ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( let mut msg = Self::new(); - // SAFETY: $minitable$ is a static of a const object. - let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; // SAFETY: // - `data.as_ptr()` is valid to read for `data.len()` @@ -148,8 +145,10 @@ void MessageClearAndParse(Context& ctx, const Descriptor& msg) { // - `msg.arena().raw()` is held for the same lifetime as `msg`. let status = unsafe { $pbr$::wire::decode( - data, msg.raw_msg(), - mini_table, msg.arena()) + data, + msg.raw_msg(), + ::MINI_TABLE, + msg.arena()) }; match status { Ok(_) => { @@ -178,13 +177,12 @@ void MessageDebug(Context& ctx, const Descriptor& msg) { return; case Kernel::kUpb: - ctx.Emit({{"minitable", UpbMinitableName(msg)}}, - R"rs( - let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; + ctx.Emit( + R"rs( let string = unsafe { $pbr$::debug_string( self.raw_msg(), - mini_table, + ::MINI_TABLE ) }; write!(f, "{}", string) @@ -298,7 +296,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { unsafe { $pbr$::upb_Message_DeepCopy( dst.inner.msg, self.msg, - $std$::ptr::addr_of!($minitable$), + ::MINI_TABLE, dst.inner.arena.raw(), ) }; dst @@ -317,11 +315,19 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { ABSL_LOG(FATAL) << "unreachable"; } -void MessageGetMinitable(Context& ctx, const Descriptor& msg) { +void UpbGeneratedMessageTraitImpls(Context& ctx, const Descriptor& msg) { if (ctx.opts().kernel == Kernel::kUpb) { ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( - pub fn raw_minitable(_private: $pbi$::Private) -> *const $pbr$::upb_MiniTable { - unsafe { $std$::ptr::addr_of!($minitable$) } + unsafe impl $pbr$::AssociatedMiniTable for $Msg$ { + const MINI_TABLE: *const $pbr$::upb_MiniTable = unsafe { $std$::ptr::addr_of!($minitable$) }; + } + + unsafe impl $pbr$::AssociatedMiniTable for $Msg$View<'_> { + const MINI_TABLE: *const $pbr$::upb_MiniTable = unsafe { $std$::ptr::addr_of!($minitable$) }; + } + + unsafe impl $pbr$::AssociatedMiniTable for $Msg$Mut<'_> { + const MINI_TABLE: *const $pbr$::upb_MiniTable = unsafe { $std$::ptr::addr_of!($minitable$) }; } )rs"); } @@ -345,17 +351,14 @@ void MessageMergeFrom(Context& ctx, const Descriptor& msg) { return; case Kernel::kUpb: ctx.Emit( - { - {"minitable", UpbMinitableName(msg)}, - }, R"rs( pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { // SAFETY: self and src are both valid `$Msg$`s. unsafe { assert!( - $pbr$::upb_Message_MergeFrom(self.raw_msg(), + $pbr$::upb_Message_MergeFrom(self.raw_msg(), src.as_view().raw_msg(), - $std$::ptr::addr_of!($minitable$), + ::MINI_TABLE, // Use a nullptr for the ExtensionRegistry. $std$::ptr::null(), self.arena().raw()) @@ -481,7 +484,6 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit( { - {"minitable", UpbMinitableName(msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, }, R"rs( @@ -559,9 +561,9 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { dest: $pb$::Mut<$pb$::Repeated>, ) { // SAFETY: - // - Elements of `src` and `dest` have message minitable `$minitable$`. + // - Elements of `src` and `dest` have message minitable `MINI_TABLE`. unsafe { - $pbr$::repeated_message_copy_from(src, dest, $std$::ptr::addr_of!($minitable$)); + $pbr$::repeated_message_copy_from(src, dest, ::MINI_TABLE); } } @@ -698,7 +700,6 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit( { - {"minitable", UpbMinitableName(msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, }, R"rs( @@ -940,7 +941,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } }}, {"into_proxied_impl", [&] { IntoProxiedForMessage(ctx, msg); }}, - {"get_upb_minitable", [&] { MessageGetMinitable(ctx, msg); }}, + {"upb_generated_message_trait_impls", + [&] { UpbGeneratedMessageTraitImpls(ctx, msg); }}, {"msg_merge_from", [&] { MessageMergeFrom(ctx, msg); }}, {"repeated_impl", [&] { MessageProxiedInRepeated(ctx, msg); }}, {"map_value_impl", [&] { MessageProxiedInMapValue(ctx, msg); }}, @@ -1155,8 +1157,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $msg_merge_from$ - $get_upb_minitable$ - $raw_arena_getter_for_msgmut$ $accessor_fns_for_muts$ @@ -1237,8 +1237,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.as_mut().merge_from(src); } - $get_upb_minitable$ - $accessor_fns$ } // impl $Msg$ @@ -1270,6 +1268,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + $upb_generated_message_trait_impls$ + extern "C" { $Msg_externs$ From 824f6ac9e95563bcf8e4e0c48f9bbc295e7e5222 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Mon, 29 Jul 2024 17:48:57 -0700 Subject: [PATCH 102/107] Migrate core (sans Parse & Serialize) and extension funcs from `protos` to `hpb` This entails: - CreateMessage - ClearMessage - CloneMessage - DeepCopy - HasExtension - ClearExtension - SetExtension - GetExtension PiperOrigin-RevId: 657386985 --- hpb/extension_lock_test.cc | 8 +- hpb/hpb.h | 125 +++++++++-------- hpb_generator/gen_messages.cc | 2 +- hpb_generator/tests/test_generated.cc | 190 +++++++++++++------------- protos/protos.h | 10 +- 5 files changed, 168 insertions(+), 167 deletions(-) diff --git a/hpb/extension_lock_test.cc b/hpb/extension_lock_test.cc index 8e5b46f021..81198082e1 100644 --- a/hpb/extension_lock_test.cc +++ b/hpb/extension_lock_test.cc @@ -36,11 +36,11 @@ std::string GenerateTestData() { model.set_str1("str"); ThemeExtension extension1; extension1.set_ext_name("theme"); - ABSL_CHECK_OK(::protos::SetExtension(&model, theme, extension1)); + ABSL_CHECK_OK(::hpb::SetExtension(&model, theme, extension1)); ThemeExtension extension2; extension2.set_ext_name("theme_extension"); - ABSL_CHECK_OK(::protos::SetExtension(&model, ThemeExtension::theme_extension, - extension2)); + ABSL_CHECK_OK( + ::hpb::SetExtension(&model, ThemeExtension::theme_extension, extension2)); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); ABSL_CHECK_OK(bytes); @@ -62,7 +62,7 @@ void TestConcurrentExtensionAccess(::hpb::ExtensionRegistry registry) { ::protos::Parse(payload, registry).value(); const auto test_main = [&] { EXPECT_EQ("str", parsed_model.str1()); }; const auto test_theme = [&] { - ASSERT_TRUE(::protos::HasExtension(&parsed_model, theme)); + ASSERT_TRUE(::hpb::HasExtension(&parsed_model, theme)); auto ext = ::protos::GetExtension(&parsed_model, theme); ASSERT_OK(ext); EXPECT_EQ((*ext)->ext_name(), "theme"); diff --git a/hpb/hpb.h b/hpb/hpb.h index b18569cc96..f6b49f5347 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -308,65 +308,6 @@ class ExtensionRegistry { upb_ExtensionRegistry* registry_; }; -} // namespace hpb - -namespace protos { -using hpb::Arena; -using hpb::ExtensionNotFoundError; -using hpb::MessageAllocationError; -using hpb::MessageDecodeError; -using hpb::MessageEncodeError; -using hpb::Ptr; -using hpb::SourceLocation; - -template -typename T::Proxy CreateMessage(::hpb::Arena& arena) { - return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), - arena.ptr()); -} - -template -void DeepCopy(Ptr source_message, Ptr target_message) { - static_assert(!std::is_const_v); - ::hpb::internal::DeepCopy( - hpb::internal::GetInternalMsg(target_message), - hpb::internal::GetInternalMsg(source_message), T::minitable(), - static_cast(target_message->GetInternalArena())); -} - -template -typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { - return ::hpb::internal::PrivateAccess::Proxy( - ::hpb::internal::DeepClone(::hpb::internal::GetInternalMsg(message), - T::minitable(), arena), - arena); -} - -template -void DeepCopy(Ptr source_message, T* target_message) { - static_assert(!std::is_const_v); - DeepCopy(source_message, Ptr(target_message)); -} - -template -void DeepCopy(const T* source_message, Ptr target_message) { - static_assert(!std::is_const_v); - DeepCopy(Ptr(source_message), target_message); -} - -template -void DeepCopy(const T* source_message, T* target_message) { - static_assert(!std::is_const_v); - DeepCopy(Ptr(source_message), Ptr(target_message)); -} - -template -void ClearMessage(hpb::internal::PtrOrRaw message) { - auto ptr = Ptr(message); - auto minitable = hpb::internal::GetMiniTable(ptr); - upb_Message_Clear(hpb::internal::GetInternalMsg(ptr), minitable); -} - template > ABSL_MUST_USE_RESULT bool HasExtension( @@ -452,7 +393,7 @@ template & id, const Extension& value) { - return ::protos::SetExtension(Ptr(message), id, value); + return ::hpb::SetExtension(Ptr(message), id, value); } template & id, Extension&& value) { - return ::protos::SetExtension(Ptr(message), id, - std::forward(value)); + return ::hpb::SetExtension(Ptr(message), id, std::forward(value)); } template & id, Ptr value) { - return ::protos::SetExtension(Ptr(message), id, value); + return ::hpb::SetExtension(Ptr(message), id, value); } template > GetExtension( return GetExtension(Ptr(message), id); } +template +typename T::Proxy CreateMessage(::hpb::Arena& arena) { + return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), + arena.ptr()); +} + +template +typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { + return ::hpb::internal::PrivateAccess::Proxy( + ::hpb::internal::DeepClone(::hpb::internal::GetInternalMsg(message), + T::minitable(), arena), + arena); +} + +template +void DeepCopy(Ptr source_message, Ptr target_message) { + static_assert(!std::is_const_v); + ::hpb::internal::DeepCopy( + hpb::internal::GetInternalMsg(target_message), + hpb::internal::GetInternalMsg(source_message), T::minitable(), + static_cast(target_message->GetInternalArena())); +} + +template +void DeepCopy(Ptr source_message, T* target_message) { + static_assert(!std::is_const_v); + DeepCopy(source_message, Ptr(target_message)); +} + +template +void DeepCopy(const T* source_message, Ptr target_message) { + static_assert(!std::is_const_v); + DeepCopy(Ptr(source_message), target_message); +} + +template +void DeepCopy(const T* source_message, T* target_message) { + static_assert(!std::is_const_v); + DeepCopy(Ptr(source_message), Ptr(target_message)); +} + +template +void ClearMessage(hpb::internal::PtrOrRaw message) { + auto ptr = Ptr(message); + auto minitable = hpb::internal::GetMiniTable(ptr); + upb_Message_Clear(hpb::internal::GetInternalMsg(ptr), minitable); +} + +} // namespace hpb + +namespace protos { +using hpb::Arena; +using hpb::ExtensionNotFoundError; +using hpb::MessageAllocationError; +using hpb::MessageDecodeError; +using hpb::MessageEncodeError; +using hpb::Ptr; +using hpb::SourceLocation; + template ABSL_MUST_USE_RESULT bool Parse(Ptr message, absl::string_view bytes) { static_assert(!std::is_const_v); diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index 2ea07e7701..de7045aac3 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -297,7 +297,7 @@ void WriteModelProxyDeclaration(const protobuf::Descriptor* descriptor, $0Proxy(upb_Message* msg, upb_Arena* arena) : internal::$0Access(($1*)msg, arena) {} - friend $0::Proxy(::protos::CreateMessage<$0>(::hpb::Arena& arena)); + friend $0::Proxy(::hpb::CreateMessage<$0>(::hpb::Arena& arena)); friend $0::Proxy(::hpb::internal::CreateMessageProxy<$0>(upb_Message*, upb_Arena*)); friend struct ::hpb::internal::PrivateAccess; diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index b09787590b..4a063e58f2 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -68,13 +68,13 @@ TEST(CppGeneratedCode, MessageEnumValue) { TEST(CppGeneratedCode, ArenaConstructor) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); EXPECT_EQ(false, testModel.has_b1()); } TEST(CppGeneratedCode, Booleans) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); EXPECT_FALSE(testModel.b1()); testModel.set_b1(true); EXPECT_TRUE(testModel.b1()); @@ -88,7 +88,7 @@ TEST(CppGeneratedCode, Booleans) { TEST(CppGeneratedCode, ScalarInt32) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test int32 defaults. EXPECT_EQ(testModel.value(), 0); EXPECT_FALSE(testModel.has_value()); @@ -131,7 +131,7 @@ TEST(CppGeneratedCode, Strings) { TEST(CppGeneratedCode, ScalarUInt32) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_uint32(), 0); EXPECT_FALSE(testModel.has_optional_uint32()); @@ -151,7 +151,7 @@ TEST(CppGeneratedCode, ScalarUInt32) { TEST(CppGeneratedCode, ScalarInt64) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_int64(), 0); EXPECT_FALSE(testModel.has_optional_int64()); @@ -175,7 +175,7 @@ TEST(CppGeneratedCode, ScalarInt64) { TEST(CppGeneratedCode, ScalarFloat) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_float(), 0.0f); EXPECT_FALSE(testModel.has_optional_float()); @@ -203,7 +203,7 @@ TEST(CppGeneratedCode, ScalarFloat) { TEST(CppGeneratedCode, ScalarDouble) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_double(), 0.0); EXPECT_FALSE(testModel.has_optional_double()); @@ -227,7 +227,7 @@ TEST(CppGeneratedCode, ScalarDouble) { TEST(CppGeneratedCode, Enums) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Check enum default value. EXPECT_EQ(TestModel_Category_IMAGES, 5); @@ -255,7 +255,7 @@ TEST(CppGeneratedCode, Enums) { TEST(CppGeneratedCode, FieldWithDefaultValue) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); EXPECT_FALSE(testModel.has_int_value_with_default()); EXPECT_EQ(testModel.int_value_with_default(), 65); @@ -270,7 +270,7 @@ TEST(CppGeneratedCode, FieldWithDefaultValue) { TEST(CppGeneratedCode, OneOfFields) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_FALSE(test_model.has_oneof_member1()); EXPECT_FALSE(test_model.has_oneof_member2()); @@ -298,7 +298,7 @@ TEST(CppGeneratedCode, OneOfFields) { TEST(CppGeneratedCode, Messages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(false, test_model.has_child_model_1()); auto child_model = test_model.child_model_1(); EXPECT_EQ(false, child_model->has_child_b1()); @@ -322,7 +322,7 @@ TEST(CppGeneratedCode, Messages) { TEST(CppGeneratedCode, NestedMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); auto nested_child = test_model.nested_child_1(); EXPECT_EQ(0, nested_child->nested_child_name().size()); auto mutable_nested_child = test_model.mutable_nested_child_1(); @@ -333,7 +333,7 @@ TEST(CppGeneratedCode, NestedMessages) { TEST(CppGeneratedCode, RepeatedMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models_size()); // Should be able to clear repeated field when empty. test_model.mutable_child_models()->clear(); @@ -362,7 +362,7 @@ TEST(CppGeneratedCode, RepeatedMessages) { TEST(CppGeneratedCode, RepeatedScalar) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.value_array_size()); // Should be able to clear repeated field when empty. test_model.mutable_value_array()->clear(); @@ -383,7 +383,7 @@ TEST(CppGeneratedCode, RepeatedScalar) { TEST(CppGeneratedCode, RepeatedFieldClear) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); test_model.mutable_value_array()->push_back(5); test_model.mutable_value_array()->push_back(16); test_model.mutable_value_array()->push_back(27); @@ -394,7 +394,7 @@ TEST(CppGeneratedCode, RepeatedFieldClear) { TEST(CppGeneratedCode, RepeatedFieldProxyForScalars) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.value_array().size()); EXPECT_EQ(0, test_model.mutable_value_array()->size()); @@ -427,7 +427,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForScalars) { TEST(CppGeneratedCode, RepeatedScalarIterator) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); test_model.mutable_value_array()->push_back(5); test_model.mutable_value_array()->push_back(16); test_model.mutable_value_array()->push_back(27); @@ -494,7 +494,7 @@ TEST(CppGeneratedCode, RepeatedScalarIterator) { TEST(CppGeneratedCode, RepeatedFieldProxyForStrings) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.repeated_string().size()); EXPECT_EQ(0, test_model.mutable_repeated_string()->size()); @@ -529,7 +529,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForStrings) { TEST(CppGeneratedCode, RepeatedFieldProxyForMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models().size()); ChildModel1 child1; child1.set_child_str1(kTestStr1); @@ -574,7 +574,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForMessages) { TEST(CppGeneratedCode, EmptyRepeatedFieldProxyForMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models().size()); ChildModel1 child1; child1.set_child_str1(kTestStr1); @@ -587,7 +587,7 @@ TEST(CppGeneratedCode, EmptyRepeatedFieldProxyForMessages) { TEST(CppGeneratedCode, RepeatedFieldProxyForMessagesIndexOperator) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models().size()); ChildModel1 child1; child1.set_child_str1(kTestStr1); @@ -605,7 +605,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForMessagesIndexOperator) { TEST(CppGeneratedCode, RepeatedStrings) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.repeated_string_size()); // Should be able to clear repeated field when empty. test_model.mutable_repeated_string()->clear(); @@ -628,11 +628,11 @@ TEST(CppGeneratedCode, MessageMapInt32KeyMessageValue) { const int key_test_value = 3; ::hpb::Arena arena; ::hpb::Arena child_arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_map_size()); test_model.clear_child_map(); EXPECT_EQ(0, test_model.child_map_size()); - auto child_model1 = ::protos::CreateMessage(child_arena); + auto child_model1 = ::hpb::CreateMessage(child_arena); child_model1.set_child_str1("abc"); test_model.set_child_map(key_test_value, child_model1); auto map_result = test_model.get_child_map(key_test_value); @@ -649,7 +649,7 @@ TEST(CppGeneratedCode, MessageMapInt32KeyMessageValue) { TEST(CppGeneratedCode, MessageMapStringKeyAndStringValue) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.str_to_str_map_size()); test_model.clear_str_to_str_map(); EXPECT_EQ(0, test_model.str_to_str_map_size()); @@ -665,7 +665,7 @@ TEST(CppGeneratedCode, MessageMapStringKeyAndStringValue) { TEST(CppGeneratedCode, MessageMapStringKeyAndInt32Value) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.str_to_int_map_size()); test_model.clear_str_to_int_map(); EXPECT_EQ(0, test_model.str_to_int_map_size()); @@ -683,26 +683,26 @@ TEST(CppGeneratedCode, MessageMapStringKeyAndInt32Value) { TEST(CppGeneratedCode, HasExtension) { TestModel model; - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); } TEST(CppGeneratedCode, HasExtensionPtr) { TestModel model; - EXPECT_EQ(false, ::protos::HasExtension(model.recursive_child(), theme)); + EXPECT_EQ(false, ::hpb::HasExtension(model.recursive_child(), theme)); } TEST(CppGeneratedCode, ClearExtensionWithEmptyExtension) { TestModel model; - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - ::protos::ClearExtension(&model, theme); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + ::hpb::ClearExtension(&model, theme); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); } TEST(CppGeneratedCode, ClearExtensionWithEmptyExtensionPtr) { TestModel model; ::hpb::Ptr recursive_child = model.mutable_recursive_child(); - ::protos::ClearExtension(recursive_child, theme); - EXPECT_EQ(false, ::protos::HasExtension(recursive_child, theme)); + ::hpb::ClearExtension(recursive_child, theme); + EXPECT_EQ(false, ::hpb::HasExtension(recursive_child, theme)); } TEST(CppGeneratedCode, SetExtension) { @@ -713,12 +713,11 @@ TEST(CppGeneratedCode, SetExtension) { ThemeExtension extension1; extension1.set_ext_name("Hello World"); prior_message = ::hpb::internal::GetInternalMsg(&extension1); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ( - true, - ::protos::SetExtension(&model, theme, std::move(extension1)).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, + ::hpb::SetExtension(&model, theme, std::move(extension1)).ok()); } - EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); EXPECT_EQ(::hpb::internal::GetInternalMsg(*ext), prior_message); @@ -726,20 +725,20 @@ TEST(CppGeneratedCode, SetExtension) { TEST(CppGeneratedCode, SetExtensionWithPtr) { ::hpb::Arena arena_model; - ::hpb::Ptr model = ::protos::CreateMessage(arena_model); + ::hpb::Ptr model = ::hpb::CreateMessage(arena_model); void* prior_message; { // Use a nested scope to make sure the arenas are fused correctly. ::hpb::Arena arena; ::hpb::Ptr extension1 = - ::protos::CreateMessage(arena); + ::hpb::CreateMessage(arena); extension1->set_ext_name("Hello World"); prior_message = ::hpb::internal::GetInternalMsg(extension1); - EXPECT_EQ(false, ::protos::HasExtension(model, theme)); - auto res = ::protos::SetExtension(model, theme, extension1); + EXPECT_EQ(false, ::hpb::HasExtension(model, theme)); + auto res = ::hpb::SetExtension(model, theme, extension1); EXPECT_EQ(true, res.ok()); } - EXPECT_EQ(true, ::protos::HasExtension(model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(model, theme)); auto ext = ::protos::GetExtension(model, theme); EXPECT_TRUE(ext.ok()); EXPECT_NE(::hpb::internal::GetInternalMsg(*ext), prior_message); @@ -748,7 +747,7 @@ TEST(CppGeneratedCode, SetExtensionWithPtr) { #ifndef _MSC_VER TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { ::hpb::Arena arena; - ::hpb::Ptr model = ::protos::CreateMessage(arena); + ::hpb::Ptr model = ::hpb::CreateMessage(arena); ThemeExtension extension1; ContainerExtension extension2; @@ -756,32 +755,32 @@ TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { return Requires(l); }; EXPECT_TRUE(canSetExtension( - [](auto p) -> decltype(::protos::SetExtension(p, theme, extension1)) {})); + [](auto p) -> decltype(::hpb::SetExtension(p, theme, extension1)) {})); // Wrong extension value type should fail to compile. EXPECT_TRUE(!canSetExtension( - [](auto p) -> decltype(::protos::SetExtension(p, theme, extension2)) {})); + [](auto p) -> decltype(::hpb::SetExtension(p, theme, extension2)) {})); // Wrong extension id with correct extension type should fail to compile. EXPECT_TRUE( - !canSetExtension([](auto p) -> decltype(::protos::SetExtension( + !canSetExtension([](auto p) -> decltype(::hpb::SetExtension( p, container_ext, extension1)) {})); } #endif TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { ::hpb::Arena arena; - ::hpb::Ptr model = ::protos::CreateMessage(arena); + ::hpb::Ptr model = ::hpb::CreateMessage(arena); void* prior_message; { // Use a nested scope to make sure the arenas are fused correctly. ::hpb::Ptr extension1 = - ::protos::CreateMessage(arena); + ::hpb::CreateMessage(arena); extension1->set_ext_name("Hello World"); prior_message = ::hpb::internal::GetInternalMsg(extension1); - EXPECT_EQ(false, ::protos::HasExtension(model, theme)); - auto res = ::protos::SetExtension(model, theme, extension1); + EXPECT_EQ(false, ::hpb::HasExtension(model, theme)); + auto res = ::hpb::SetExtension(model, theme, extension1); EXPECT_EQ(true, res.ok()); } - EXPECT_EQ(true, ::protos::HasExtension(model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(model, theme)); auto ext = ::protos::GetExtension(model, theme); EXPECT_TRUE(ext.ok()); EXPECT_NE(::hpb::internal::GetInternalMsg(*ext), prior_message); @@ -792,16 +791,16 @@ TEST(CppGeneratedCode, SetExtensionFusingFailureShouldCopy) { char initial_block[1000]; hpb::Arena arena(initial_block, sizeof(initial_block)); - hpb::Ptr model = ::protos::CreateMessage(arena); + hpb::Ptr model = ::hpb::CreateMessage(arena); ThemeExtension extension1; extension1.set_ext_name("Hello World"); ASSERT_FALSE( upb_Arena_Fuse(arena.ptr(), ::hpb::internal::GetArena(&extension1))); - EXPECT_FALSE(::protos::HasExtension(model, theme)); - auto status = ::protos::SetExtension(model, theme, std::move(extension1)); + EXPECT_FALSE(::hpb::HasExtension(model, theme)); + auto status = ::hpb::SetExtension(model, theme, std::move(extension1)); EXPECT_TRUE(status.ok()); - EXPECT_TRUE(::protos::HasExtension(model, theme)); + EXPECT_TRUE(::hpb::HasExtension(model, theme)); EXPECT_TRUE(::protos::GetExtension(model, theme).ok()); } @@ -809,10 +808,10 @@ TEST(CppGeneratedCode, SetExtensionShouldClone) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); extension1.set_ext_name("Goodbye"); - EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); EXPECT_EQ((*ext)->ext_name(), "Hello World"); @@ -822,12 +821,11 @@ TEST(CppGeneratedCode, SetExtensionShouldCloneConst) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ( - true, - ::protos::SetExtension(&model, theme, std::as_const(extension1)).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, + ::hpb::SetExtension(&model, theme, std::as_const(extension1)).ok()); extension1.set_ext_name("Goodbye"); - EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); EXPECT_EQ((*ext)->ext_name(), "Hello World"); @@ -837,21 +835,19 @@ TEST(CppGeneratedCode, SetExtensionOnMutableChild) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, - ::protos::HasExtension(model.mutable_recursive_child(), theme)); - EXPECT_EQ(true, ::protos::SetExtension(model.mutable_recursive_child(), theme, - extension1) + EXPECT_EQ(false, ::hpb::HasExtension(model.mutable_recursive_child(), theme)); + EXPECT_EQ(true, ::hpb::SetExtension(model.mutable_recursive_child(), theme, + extension1) .ok()); - EXPECT_EQ(true, - ::protos::HasExtension(model.mutable_recursive_child(), theme)); + EXPECT_EQ(true, ::hpb::HasExtension(model.mutable_recursive_child(), theme)); } TEST(CppGeneratedCode, GetExtension) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); EXPECT_EQ("Hello World", ::protos::GetExtension(&model, theme).value()->ext_name()); } @@ -862,10 +858,10 @@ TEST(CppGeneratedCode, GetExtensionOnMutableChild) { extension1.set_ext_name("Hello World"); ::hpb::Ptr mutable_recursive_child = model.mutable_recursive_child(); - EXPECT_EQ(false, ::protos::HasExtension(mutable_recursive_child, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(mutable_recursive_child, theme)); EXPECT_EQ( true, - ::protos::SetExtension(mutable_recursive_child, theme, extension1).ok()); + ::hpb::SetExtension(mutable_recursive_child, theme, extension1).ok()); EXPECT_EQ("Hello World", ::protos::GetExtension(mutable_recursive_child, theme) .value() @@ -878,10 +874,10 @@ TEST(CppGeneratedCode, GetExtensionOnImmutableChild) { extension1.set_ext_name("Hello World"); ::hpb::Ptr mutable_recursive_child = model.mutable_recursive_child(); - EXPECT_EQ(false, ::protos::HasExtension(mutable_recursive_child, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(mutable_recursive_child, theme)); EXPECT_EQ( true, - ::protos::SetExtension(mutable_recursive_child, theme, extension1).ok()); + ::hpb::SetExtension(mutable_recursive_child, theme, extension1).ok()); ::hpb::Ptr recursive_child = model.recursive_child(); EXPECT_EQ("Hello World", ::protos::GetExtension(recursive_child, theme).value()->ext_name()); @@ -899,8 +895,7 @@ TEST(CppGeneratedCode, SerializeUsingArena) { TEST(CppGeneratedCode, SerializeProxyUsingArena) { ::upb::Arena message_arena; - TestModel::Proxy model_proxy = - ::protos::CreateMessage(message_arena); + TestModel::Proxy model_proxy = ::hpb::CreateMessage(message_arena); model_proxy.set_str1("Hello World"); ::upb::Arena arena; absl::StatusOr bytes = @@ -926,7 +921,7 @@ TEST(CppGeneratedCode, Parse) { model.set_str1("Test123"); ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); @@ -940,12 +935,11 @@ TEST(CppGeneratedCode, ParseIntoPtrToModel) { model.set_str1("Test123"); ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); - ::hpb::Ptr parsed_model = - ::protos::CreateMessage(arena); + ::hpb::Ptr parsed_model = ::hpb::CreateMessage(arena); EXPECT_TRUE(::protos::Parse(parsed_model, bytes.value())); EXPECT_EQ("Test123", parsed_model->str1()); // Should return an extension even if we don't pass ExtensionRegistry @@ -958,9 +952,9 @@ TEST(CppGeneratedCode, ParseWithExtensionRegistry) { model.set_str1("Test123"); ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); - EXPECT_EQ(true, ::protos::SetExtension( - &model, ThemeExtension::theme_extension, extension1) + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(true, ::hpb::SetExtension(&model, ThemeExtension::theme_extension, + extension1) .ok()); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); @@ -1129,10 +1123,10 @@ TEST(CppGeneratedCode, ClearSubMessage) { new_child->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - EXPECT_TRUE(::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_TRUE(::hpb::SetExtension(&model, theme, extension1).ok()); EXPECT_TRUE(model.mutable_child_model_1()->has_child_str1()); // Clear using Ptr - ::protos::ClearMessage(model.mutable_child_model_1()); + ::hpb::ClearMessage(model.mutable_child_model_1()); EXPECT_FALSE(model.mutable_child_model_1()->has_child_str1()); } @@ -1146,14 +1140,14 @@ TEST(CppGeneratedCode, ClearMessage) { new_child.value()->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - EXPECT_TRUE(::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_TRUE(::hpb::SetExtension(&model, theme, extension1).ok()); // Clear using T* - ::protos::ClearMessage(&model); + ::hpb::ClearMessage(&model); // Verify that scalars, repeated fields and extensions are cleared. EXPECT_FALSE(model.has_int64()); EXPECT_FALSE(model.has_str2()); EXPECT_TRUE(model.child_models().empty()); - EXPECT_FALSE(::protos::HasExtension(&model, theme)); + EXPECT_FALSE(::hpb::HasExtension(&model, theme)); } TEST(CppGeneratedCode, CanInvokeClearMessageWithPtr) { @@ -1163,7 +1157,7 @@ TEST(CppGeneratedCode, CanInvokeClearMessageWithPtr) { auto new_child = model.add_child_models(); // Clear using Ptr auto ptr = ::hpb::Ptr(&model); - ::protos::ClearMessage(ptr); + ::hpb::ClearMessage(ptr); // Successful clear EXPECT_FALSE(model.has_int64()); } @@ -1174,14 +1168,14 @@ TEST(CppGeneratedCode, CanInvokeClearMessageWithRawPtr) { model.set_int64(5); auto new_child = model.add_child_models(); // Clear using T* - ::protos::ClearMessage(&model); + ::hpb::ClearMessage(&model); // Successful clear EXPECT_FALSE(model.has_int64()); } template bool CanCallClearMessage() { - return Requires([](auto x) -> decltype(::protos::ClearMessage(x)) {}); + return Requires([](auto x) -> decltype(::hpb::ClearMessage(x)) {}); } TEST(CppGeneratedCode, CannotInvokeClearMessageWithConstPtr) { @@ -1204,13 +1198,13 @@ TEST(CppGeneratedCode, DeepCopy) { new_child.value()->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - EXPECT_TRUE(::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_TRUE(::hpb::SetExtension(&model, theme, extension1).ok()); TestModel target; target.set_b1(true); - ::protos::DeepCopy(&model, &target); + ::hpb::DeepCopy(&model, &target); EXPECT_FALSE(target.b1()) << "Target was not cleared before copying content "; EXPECT_EQ(target.str2(), "Hello"); - EXPECT_TRUE(::protos::HasExtension(&target, theme)); + EXPECT_TRUE(::hpb::HasExtension(&target, theme)); } TEST(CppGeneratedCode, HasExtensionAndRegistry) { @@ -1223,7 +1217,7 @@ TEST(CppGeneratedCode, HasExtensionAndRegistry) { new_child.value()->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - ASSERT_TRUE(::protos::SetExtension(&source, theme, extension1).ok()); + ASSERT_TRUE(::hpb::SetExtension(&source, theme, extension1).ok()); // Now that we have a source model with extension data, serialize. ::hpb::Arena arena; @@ -1232,7 +1226,7 @@ TEST(CppGeneratedCode, HasExtensionAndRegistry) { // Test with ExtensionRegistry ::hpb::ExtensionRegistry extensions({&theme}, arena); TestModel parsed_model = ::protos::Parse(data, extensions).value(); - EXPECT_TRUE(::protos::HasExtension(&parsed_model, theme)); + EXPECT_TRUE(::hpb::HasExtension(&parsed_model, theme)); } TEST(CppGeneratedCode, FieldNumberConstants) { diff --git a/protos/protos.h b/protos/protos.h index 250152d41c..4d61918050 100644 --- a/protos/protos.h +++ b/protos/protos.h @@ -4,7 +4,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd - #ifndef UPB_PROTOS_PROTOS_H_ #define UPB_PROTOS_PROTOS_H_ #include "google/protobuf/hpb/hpb.h" @@ -26,5 +25,14 @@ using hpb::internal::PrivateAccess; using hpb::internal::Serialize; using hpb::internal::SetExtension; } // namespace internal +using hpb::ClearMessage; +using hpb::CloneMessage; +using hpb::CreateMessage; +using hpb::DeepCopy; + +using hpb::ClearExtension; +using hpb::GetExtension; +using hpb::HasExtension; +using hpb::SetExtension; } // namespace protos #endif From c7d469748188a17716f1de5a025324db933f5730 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 30 Jul 2024 05:39:53 -0700 Subject: [PATCH 103/107] Add an protobuf::__internal::SealedInternal trait The purpose of this trait is that it is declared as a supertrait of the traits that we don't want application code to implement (like "Proxied" and "MessageView"); application code can see those traits but not the supertrait, which enables them to use them but not implement them. PiperOrigin-RevId: 657555025 --- rust/codegen_traits.rs | 23 ++++++++++------ rust/cord.rs | 6 +++- rust/internal.rs | 8 ++++++ rust/map.rs | 8 +++++- rust/primitive.rs | 3 ++ rust/proxied.rs | 29 +++++++++++++------- rust/repeated.rs | 8 +++++- rust/string.rs | 12 +++++++- rust/upb.rs | 4 ++- src/google/protobuf/compiler/rust/enum.cc | 2 ++ src/google/protobuf/compiler/rust/message.cc | 6 ++++ 11 files changed, 86 insertions(+), 23 deletions(-) diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index ad8dd64a87..3cd94df6dd 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -7,6 +7,7 @@ //! Traits that are implemeted by codegen types. +use crate::__internal::SealedInternal; use crate::{MutProxied, MutProxy, ViewProxy}; use create::Parse; use read::Serialize; @@ -14,7 +15,8 @@ use std::fmt::Debug; use write::{Clear, ClearAndParse}; /// A trait that all generated owned message types implement. -pub trait Message: MutProxied +pub trait Message: SealedInternal + + MutProxied // Create traits: + Parse + Default // Read traits: @@ -28,7 +30,8 @@ pub trait Message: MutProxied {} /// A trait that all generated message views implement. -pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> +pub trait MessageView<'msg>: SealedInternal + + ViewProxy<'msg, Proxied = Self::Message> // Read traits: + Debug + Serialize // Thread safety: @@ -41,8 +44,8 @@ pub trait MessageView<'msg>: ViewProxy<'msg, Proxied = Self::Message> } /// A trait that all generated message muts implement. -pub trait MessageMut<'msg>: - MutProxy<'msg, MutProxied = Self::Message> +pub trait MessageMut<'msg>: SealedInternal + + MutProxy<'msg, MutProxied = Self::Message> // Read traits: + Debug + Serialize // Write traits: @@ -60,7 +63,8 @@ pub trait MessageMut<'msg>: /// Operations related to constructing a message. Only owned messages implement /// these traits. pub(crate) mod create { - pub trait Parse: Sized { + use super::SealedInternal; + pub trait Parse: SealedInternal + Sized { fn parse(serialized: &[u8]) -> Result; } } @@ -69,7 +73,9 @@ pub(crate) mod create { /// have a `&self` receiver on an owned message). Owned messages, views, and /// muts all implement these traits. pub(crate) mod read { - pub trait Serialize { + use super::SealedInternal; + + pub trait Serialize: SealedInternal { fn serialize(&self) -> Result, crate::SerializeError>; } } @@ -78,12 +84,13 @@ pub(crate) mod read { /// self` receiver on an owned message). Owned messages and muts implement these /// traits. pub(crate) mod write { + use super::SealedInternal; - pub trait Clear { + pub trait Clear: SealedInternal { fn clear(&mut self); } - pub trait ClearAndParse { + pub trait ClearAndParse: SealedInternal { fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), crate::ParseError>; } } diff --git a/rust/cord.rs b/rust/cord.rs index 14c9a60991..19fdb0ba5a 100644 --- a/rust/cord.rs +++ b/rust/cord.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::__internal::Private; +use crate::__internal::{Private, SealedInternal}; use crate::{ AsView, IntoProxied, IntoView, ProtoBytes, ProtoStr, ProtoString, Proxied, Proxy, View, ViewProxy, @@ -26,6 +26,10 @@ macro_rules! impl_cord_types { Owned($t), } + impl SealedInternal for [< $t Cord>] {} + + impl<'msg> SealedInternal for [< $t Cow>]<'msg> {} + impl Proxied for [< $t Cord>] { type View<'msg> = [< $t Cow>]<'msg>; } diff --git a/rust/internal.rs b/rust/internal.rs index c2bdd007c8..a2397bdbe7 100644 --- a/rust/internal.rs +++ b/rust/internal.rs @@ -23,3 +23,11 @@ pub use crate::__runtime::{PtrAndLen, RawMap, RawMessage, RawRepeatedField}; /// Used to protect internal-only items from being used accidentally. #[derive(Debug)] pub struct Private; + +/// A trait that is used as a subtrait of traits that we intend to be used but +/// not be implemented by users. +/// +/// This is slightly less 'sealed' than the typical sealed trait pattern would +/// permit in other crates; this trait is intended to be available to crates +/// which were generated by protoc, but not to application code. +pub trait SealedInternal {} diff --git a/rust/map.rs b/rust/map.rs index 57647d649b..ec72d9a49e 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -8,7 +8,7 @@ use crate::{ AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, - __internal::Private, + __internal::{Private, SealedInternal}, __runtime::{InnerMap, InnerMapMut, RawMap, RawMapIter}, }; use std::marker::PhantomData; @@ -72,6 +72,8 @@ unsafe impl> Sync for Map {} // it does not use thread-local data or similar. unsafe impl> Send for Map {} +impl> SealedInternal for Map {} + impl> Drop for Map { fn drop(&mut self) { // SAFETY: @@ -129,6 +131,8 @@ impl> AsMut for Map { } } +impl<'msg, K: Proxied, V: ProxiedInMapValue> SealedInternal for MapView<'msg, K, V> {} + impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, V> {} impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapView<'msg, K, V> { @@ -150,6 +154,8 @@ impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoView<'msg> for MapView<'msg, impl<'msg, K: Proxied, V: ProxiedInMapValue> ViewProxy<'msg> for MapView<'msg, K, V> {} +impl<'msg, K: Proxied, V: ProxiedInMapValue> SealedInternal for MapMut<'msg, K, V> {} + impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapMut<'msg, K, V> {} impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapMut<'msg, K, V> { diff --git a/rust/primitive.rs b/rust/primitive.rs index a87f3305f0..bf5af332a9 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -4,11 +4,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +use crate::__internal::SealedInternal; use crate::{AsView, IntoView, Proxied, Proxy, ViewProxy}; macro_rules! impl_singular_primitives { ($($t:ty),*) => { $( + impl SealedInternal for $t {} + impl Proxied for $t { type View<'msg> = $t; } diff --git a/rust/proxied.rs b/rust/proxied.rs index c56b18544f..32147908b5 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -44,7 +44,7 @@ //! implemented the concept of "proxy" types. Proxy types are a reference-like //! indirection between the user and the internal memory representation. -use crate::__internal::Private; +use crate::__internal::{Private, SealedInternal}; use std::fmt::Debug; /// A type that can be accessed through a reference-like proxy. @@ -52,7 +52,7 @@ use std::fmt::Debug; /// An instance of a `Proxied` can be accessed immutably via `Proxied::View`. /// /// All Protobuf field types implement `Proxied`. -pub trait Proxied: AsView + Sized { +pub trait Proxied: SealedInternal + AsView + Sized { /// The proxy type that provides shared access to a `T`, like a `&'msg T`. /// /// Most code should use the type alias [`View`]. @@ -67,7 +67,7 @@ pub trait Proxied: AsView + Sized { /// and immutably via `MutProxied::View`. /// /// `MutProxied` is implemented by message, map and repeated field types. -pub trait MutProxied: Proxied + AsMut { +pub trait MutProxied: SealedInternal + Proxied + AsMut { /// The proxy type that provides exclusive mutable access to a `T`, like a /// `&'msg mut T`. /// @@ -95,7 +95,7 @@ pub type Mut<'msg, T> = ::Mut<'msg>; /// types. /// /// On ViewProxy this will behave as a reborrow into a shorter lifetime. -pub trait AsView { +pub trait AsView: SealedInternal { type Proxied: Proxied; /// Converts a borrow into a `View` with the lifetime of that borrow. @@ -127,7 +127,7 @@ pub trait AsView { /// /// On a ViewProxy this will behave as a reborrow into a shorter lifetime /// (semantically matching a `&'a T` into a `&'b T` where `'a: 'b`). -pub trait IntoView<'msg>: AsView { +pub trait IntoView<'msg>: SealedInternal + AsView { /// Converts into a `View` with a potentially shorter lifetime. /// /// In non-generic code we don't need to use `into_view` because the proxy @@ -162,7 +162,7 @@ pub trait IntoView<'msg>: AsView { /// implemented on both owned `Proxied` types as well as MutProxy types. /// /// On MutProxy this will behave as a reborrow into a shorter lifetime. -pub trait AsMut: AsView { +pub trait AsMut: SealedInternal + AsView { type MutProxied: MutProxied; /// Converts a borrow into a `Mut` with the lifetime of that borrow. @@ -173,7 +173,7 @@ pub trait AsMut: AsView { /// /// On a MutProxy this will behave as a reborrow into a shorter lifetime /// (semantically matching a `&mut 'a T` into a `&mut 'b T` where `'a: 'b`). -pub trait IntoMut<'msg>: AsMut { +pub trait IntoMut<'msg>: SealedInternal + AsMut { /// Converts into a `Mut` with a potentially shorter lifetime. /// /// In non-generic code we don't need to use `into_mut` because the proxy @@ -206,16 +206,19 @@ pub trait IntoMut<'msg>: AsMut { /// /// This trait is intentionally made non-object-safe to prevent a potential /// future incompatible change. -pub trait Proxy<'msg>: 'msg + IntoView<'msg> + Sync + Unpin + Sized + Debug {} +pub trait Proxy<'msg>: + SealedInternal + 'msg + IntoView<'msg> + Sync + Unpin + Sized + Debug +{ +} /// Declares conversion operations common to view proxies. -pub trait ViewProxy<'msg>: Proxy<'msg> + Send {} +pub trait ViewProxy<'msg>: SealedInternal + Proxy<'msg> + Send {} /// Declares operations common to all mut proxies. /// /// This trait is intentionally made non-object-safe to prevent a potential /// future incompatible change. -pub trait MutProxy<'msg>: Proxy<'msg> + AsMut + IntoMut<'msg> { +pub trait MutProxy<'msg>: SealedInternal + Proxy<'msg> + AsMut + IntoMut<'msg> { /// Gets an immutable view of this field. This is shorthand for `as_view`. /// /// This provides a shorter lifetime than `into_view` but can also be called @@ -265,6 +268,8 @@ mod tests { } } + impl SealedInternal for MyProxied {} + impl Proxied for MyProxied { type View<'msg> = MyProxiedView<'msg>; } @@ -292,6 +297,8 @@ mod tests { my_proxied_ref: &'msg MyProxied, } + impl<'msg> SealedInternal for MyProxiedView<'msg> {} + impl MyProxiedView<'_> { fn val(&self) -> &str { &self.my_proxied_ref.val @@ -324,6 +331,8 @@ mod tests { my_proxied_ref: &'msg mut MyProxied, } + impl<'msg> SealedInternal for MyProxiedMut<'msg> {} + impl<'msg> Proxy<'msg> for MyProxiedMut<'msg> {} impl<'msg> AsView for MyProxiedMut<'msg> { diff --git a/rust/repeated.rs b/rust/repeated.rs index 1a43dceab0..1fd5e0e364 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -17,7 +17,7 @@ use std::marker::PhantomData; use crate::{ AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, View, ViewProxy, - __internal::Private, + __internal::{Private, SealedInternal}, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -382,6 +382,8 @@ where type View<'msg> = RepeatedView<'msg, T> where Repeated: 'msg; } +impl SealedInternal for Repeated where T: ProxiedInRepeated {} + impl AsView for Repeated where T: ProxiedInRepeated, @@ -411,6 +413,8 @@ where } } +impl<'msg, T> SealedInternal for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} + impl<'msg, T> Proxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} impl<'msg, T> AsView for RepeatedView<'msg, T> @@ -440,6 +444,8 @@ where impl<'msg, T> ViewProxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} +impl<'msg, T> SealedInternal for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} + impl<'msg, T> Proxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} impl<'msg, T> AsView for RepeatedMut<'msg, T> diff --git a/rust/string.rs b/rust/string.rs index 32421e53f3..7c758783be 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -9,7 +9,7 @@ #![allow(dead_code)] #![allow(unused)] -use crate::__internal::Private; +use crate::__internal::{Private, SealedInternal}; use crate::__runtime::{InnerProtoString, PtrAndLen, RawMessage}; use crate::{ utf8::Utf8Chunks, AsView, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Optional, Proxied, @@ -67,6 +67,8 @@ impl From<&[u8; N]> for ProtoBytes { } } +impl SealedInternal for ProtoBytes {} + impl Proxied for ProtoBytes { type View<'msg> = &'msg [u8]; } @@ -127,6 +129,8 @@ impl IntoProxied for Arc<[u8]> { } } +impl SealedInternal for &[u8] {} + impl<'msg> Proxy<'msg> for &'msg [u8] {} impl AsView for &[u8] { @@ -217,6 +221,8 @@ impl ProtoString { } } +impl SealedInternal for ProtoString {} + impl AsRef<[u8]> for ProtoString { fn as_ref(&self) -> &[u8] { self.inner.as_bytes() @@ -241,6 +247,10 @@ impl From<&[u8]> for ProtoString { } } +impl SealedInternal for &str {} + +impl SealedInternal for &ProtoStr {} + impl IntoProxied for &str { fn into_proxied(self, _private: Private) -> ProtoString { ProtoString::from(self) diff --git a/rust/upb.rs b/rust/upb.rs index b90de06d41..cc4db52bac 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -7,7 +7,7 @@ //! UPB FFI wrapper code for use by Rust Protobuf. -use crate::__internal::{Enum, Private}; +use crate::__internal::{Enum, Private, SealedInternal}; use crate::{ IntoProxied, Map, MapIter, MapMut, MapView, Mut, ProtoBytes, ProtoStr, ProtoString, Proxied, ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, @@ -64,6 +64,8 @@ impl ScratchSpace { #[doc(hidden)] pub type SerializedData = upb::OwnedArenaBox<[u8]>; +impl SealedInternal for SerializedData {} + impl IntoProxied for SerializedData { fn into_proxied(self, _private: Private) -> ProtoBytes { ProtoBytes { inner: InnerProtoString(self) } diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 8c39cb8602..9b4a4402cf 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -395,6 +395,8 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } } + impl $pbi$::SealedInternal for $name$ {} + impl $pb$::Proxied for $name$ { type View<'a> = $name$; } diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 6b652f4af9..8f3cafaee1 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -1018,6 +1018,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { type View<'msg> = $Msg$View<'msg>; } + impl $pbi$::SealedInternal for $Msg$ {} + impl $pb$::MutProxied for $Msg$ { type Mut<'msg> = $Msg$Mut<'msg>; } @@ -1029,6 +1031,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { _phantom: $Phantom$<&'msg ()>, } + impl<'msg> $pbi$::SealedInternal for $Msg$View<'msg> {} + impl<'msg> $pb$::MessageView<'msg> for $Msg$View<'msg> { type Message = $Msg$; } @@ -1101,6 +1105,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { inner: $pbr$::MutatorMessageRef<'msg>, } + impl<'msg> $pbi$::SealedInternal for $Msg$Mut<'msg> {} + impl<'msg> $pb$::MessageMut<'msg> for $Msg$Mut<'msg> { type Message = $Msg$; } From 9cceb6278dc86e972fd8d7e108d2acaa0c2d9c27 Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Tue, 30 Jul 2024 08:59:24 -0700 Subject: [PATCH 104/107] Fix broken github_only comment in cargo build rule PiperOrigin-RevId: 657611564 --- rust/BUILD | 92 +++++++++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/rust/BUILD b/rust/BUILD index 43b6e6a49d..31c8c800d5 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -211,55 +211,55 @@ config_setting( ) # begin:github_only -# pkg_files( -# name = "rust_protobuf_src", -# srcs = glob( -# [ -# "*", -# ], -# ), -# strip_prefix = strip_prefix.from_root("rust"), -# visibility = ["//:__pkg__"], -# ) +pkg_files( + name = "rust_protobuf_src", + srcs = glob( + [ + "*", + ], + ), + strip_prefix = strip_prefix.from_root("rust"), + visibility = ["//:__pkg__"], +) -# pkg_files( -# name = "crate_root_files", -# srcs = glob(["cargo/**/*"]), -# strip_prefix = strip_prefix.from_root("rust/cargo"), -# visibility = ["//:__pkg__"], -# ) +pkg_files( + name = "crate_root_files", + srcs = glob(["cargo/**/*"]), + strip_prefix = strip_prefix.from_root("rust/cargo"), + visibility = ["//:__pkg__"], +) -# pkg_filegroup( -# name = "rust_protobuf_src_dir", -# srcs = [ -# ":rust_protobuf_src", -# "//rust/upb:rust_protobuf_upb_src", -# ], -# prefix = "src", -# ) +pkg_filegroup( + name = "rust_protobuf_src_dir", + srcs = [ + ":rust_protobuf_src", + "//rust/upb:rust_protobuf_upb_src", + ], + prefix = "src", +) -# pkg_files( -# name = "amalgamated_upb", -# srcs = ["//upb:gen_amalgamation"], -# strip_prefix = strip_prefix.from_root(""), -# ) +pkg_files( + name = "amalgamated_upb", + srcs = ["//upb:gen_amalgamation"], + strip_prefix = strip_prefix.from_root(""), +) -# pkg_filegroup( -# name = "rust_protobuf_libupb_src", -# srcs = [ -# ":amalgamated_upb", -# "//upb/cmake:upb_cmake_dist", -# ], -# prefix = "libupb", -# ) +pkg_filegroup( + name = "rust_protobuf_libupb_src", + srcs = [ + ":amalgamated_upb", + "//upb/cmake:upb_cmake_dist", + ], + prefix = "libupb", +) -# pkg_zip( -# name = "rust_crate", -# srcs = [ -# ":crate_root_files", -# ":rust_protobuf_libupb_src", -# ":rust_protobuf_src_dir", -# "//:LICENSE", -# ], -# ) +pkg_zip( + name = "rust_crate", + srcs = [ + ":crate_root_files", + ":rust_protobuf_libupb_src", + ":rust_protobuf_src_dir", + "//:LICENSE", + ], +) # end:github_only From 76c767fa1c932b41d909831766293e7364607916 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 30 Jul 2024 09:17:24 -0700 Subject: [PATCH 105/107] Add traits for interop fns. The owned and mut interop traits have the corresponding to/from behaviors on cpp but are defined as empty on upb, while the view interop is implemented for both. PiperOrigin-RevId: 657617187 --- rust/codegen_traits.rs | 85 +++++++++++++++++++- rust/shared.rs | 1 + rust/test/cpp/interop/main.rs | 27 +++---- src/google/protobuf/compiler/rust/message.cc | 54 ++++++++++++- 4 files changed, 147 insertions(+), 20 deletions(-) diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index 3cd94df6dd..d9a39411f8 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -8,8 +8,10 @@ //! Traits that are implemeted by codegen types. use crate::__internal::SealedInternal; +use crate::__runtime::RawMessage; use crate::{MutProxied, MutProxy, ViewProxy}; use create::Parse; +use interop::{MessageMutInterop, MessageViewInterop, OwnedMessageInterop}; use read::Serialize; use std::fmt::Debug; use write::{Clear, ClearAndParse}; @@ -27,7 +29,10 @@ pub trait Message: SealedInternal + Send + Sync // Copy/Clone: + Clone - {} + // C++ Interop: + + OwnedMessageInterop +{ +} /// A trait that all generated message views implement. pub trait MessageView<'msg>: SealedInternal @@ -38,6 +43,8 @@ pub trait MessageView<'msg>: SealedInternal + Send + Sync // Copy/Clone: + Copy + Clone + // C++ Interop: + + MessageViewInterop<'msg> { #[doc(hidden)] type Message: Message; @@ -55,6 +62,8 @@ pub trait MessageMut<'msg>: SealedInternal + Sync // Copy/Clone: // (Neither) + // C++ Interop: + + MessageMutInterop<'msg> { #[doc(hidden)] type Message: Message; @@ -94,3 +103,77 @@ pub(crate) mod write { fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), crate::ParseError>; } } + +/// Traits related to interop with C or C++. +/// +/// These traits are deliberately not available on the prelude, as they should +/// be used rarely and with great care. +pub(crate) mod interop { + use super::{RawMessage, SealedInternal}; + + /// Traits related to owned message interop. Note that these trait fns + /// are only available on C++ kernel as upb interop of owned messages + /// requires more work to handle the Arena behavior. + pub trait OwnedMessageInterop: SealedInternal { + /// Drops `self` and returns the `RawMessage` that it was wrapping + /// without deleting it. + /// + /// The caller is responsible for ensuring the returned RawMessage is + /// subsequently deleted (eg by moving it into a std::unique_ptr in + /// C++), or else it will leak. + #[cfg(cpp_kernel)] + fn __unstable_leak_raw_message(self) -> RawMessage; + + /// Takes exclusive ownership of the `raw_message`. + /// + /// # Safety + /// - The underlying message must be for the same type as `Self` + /// - The pointer passed in must not be used by the caller after being + /// passed here (must not be read, written, or deleted) + #[cfg(cpp_kernel)] + unsafe fn __unstable_take_ownership_of_raw_message(raw_message: RawMessage) -> Self; + } + + /// Traits related to message view interop. + pub trait MessageViewInterop<'msg>: SealedInternal { + /// Borrows `self` as an underlying `RawMessage`. + /// + /// Note that the returned Value must be used under the same constraints + /// as though it were a borrow of `self`: it should be treated as a + /// `const Message*` in C++, and not be mutated in any way, and any + /// mutation to the parent message may invalidate it, and it + /// must not be deleted. + fn __unstable_as_raw_message(&self) -> RawMessage; + + /// Wraps the provided `RawMessage` as a MessageView. + /// + /// # Safety + /// - The underlying message must be for the same type as `Self` + /// - The underlying message must be alive for 'msg and not mutated + /// while the wrapper is live. + unsafe fn __unstable_wrap_raw_message(raw: &'msg RawMessage) -> Self; + } + + /// Traits related to message mut interop. Note that these trait fns + /// are only available on C++ kernel as upb interop of owned messages + /// requires more work to handle the Arena behavior. + pub trait MessageMutInterop<'msg>: SealedInternal { + /// Exclusive borrows `self` as a `RawMessage`. + /// + /// Note that the returned Value must be used under the same constraints + /// as though it were a mut borrow of `self`: it should be treated as a + /// non-owned `Message*` in C++. And any mutation to the parent message + /// may invalidate it, and it must not be deleted. + #[cfg(cpp_kernel)] + fn __unstable_as_raw_message_mut(&mut self) -> RawMessage; + + /// Wraps the provided `RawMessage` as a MessageMut. + /// + /// # Safety + /// - The underlying message must be for the same type as `Self` + /// - The underlying message must be alive for 'msg and not read or + /// mutated while the wrapper is live. + #[cfg(cpp_kernel)] + unsafe fn __unstable_wrap_raw_message_mut(raw: &'msg mut RawMessage) -> Self; + } +} diff --git a/rust/shared.rs b/rust/shared.rs index 80dd3827dd..59e7bd8205 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -24,6 +24,7 @@ use std::fmt; pub mod __public { pub use crate::codegen_traits::{ create::Parse, + interop::{MessageMutInterop, MessageViewInterop, OwnedMessageInterop}, read::Serialize, write::{Clear, ClearAndParse}, Message, MessageMut, MessageView, diff --git a/rust/test/cpp/interop/main.rs b/rust/test/cpp/interop/main.rs index 7c9a4dda55..467f4559d0 100644 --- a/rust/test/cpp/interop/main.rs +++ b/rust/test/cpp/interop/main.rs @@ -9,6 +9,7 @@ use googletest::prelude::*; use protobuf_cpp::prelude::*; use protobuf_cpp::__runtime::{PtrAndLen, RawMessage}; +use protobuf_cpp::{MessageMutInterop, MessageViewInterop, OwnedMessageInterop}; use unittest_rust_proto::{TestAllExtensions, TestAllTypes, TestAllTypesMut, TestAllTypesView}; macro_rules! proto_assert_eq { @@ -39,9 +40,7 @@ extern "C" { fn send_to_cpp() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int32(7); - let i = unsafe { - TakeOwnershipAndGetOptionalInt32(msg1.__unstable_leak_cpp_repr_grant_permission_to_break()) - }; + let i = unsafe { TakeOwnershipAndGetOptionalInt32(msg1.__unstable_leak_raw_message()) }; assert_eq!(i, 7); } @@ -49,7 +48,7 @@ fn send_to_cpp() { fn mutate_message_mut_in_cpp() { let mut msg1 = TestAllTypes::new(); unsafe { - MutateTestAllTypes(msg1.as_mut().__unstable_cpp_repr_grant_permission_to_break()); + MutateTestAllTypes(msg1.as_mut().__unstable_as_raw_message_mut()); } let mut msg2 = TestAllTypes::new(); @@ -65,9 +64,7 @@ fn deserialize_in_rust() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); msg1.set_optional_bytes(b"some cool data I guess"); - let serialized = unsafe { - SerializeTestAllTypes(msg1.as_view().__unstable_cpp_repr_grant_permission_to_break()) - }; + let serialized = unsafe { SerializeTestAllTypes(msg1.as_view().__unstable_as_raw_message()) }; let msg2 = TestAllTypes::parse(&serialized).unwrap(); proto_assert_eq!(msg1, msg2); @@ -81,7 +78,7 @@ fn deserialize_in_cpp() { let data = msg1.serialize().unwrap(); let msg2 = unsafe { - TestAllTypes::__unstable_wrap_cpp_grant_permission_to_break(DeserializeTestAllTypes( + TestAllTypes::__unstable_take_ownership_of_raw_message(DeserializeTestAllTypes( (*data).as_ptr(), data.len(), )) @@ -98,7 +95,7 @@ fn deserialize_in_cpp_into_mut() { let data = msg1.serialize().unwrap(); let mut raw_msg = unsafe { DeserializeTestAllTypes((*data).as_ptr(), data.len()) }; - let msg2 = TestAllTypesMut::__unstable_wrap_cpp_grant_permission_to_break(&mut raw_msg); + let msg2 = unsafe { TestAllTypesMut::__unstable_wrap_raw_message_mut(&mut raw_msg) }; proto_assert_eq!(msg1, msg2); @@ -116,7 +113,7 @@ fn deserialize_in_cpp_into_view() { let data = msg1.serialize().unwrap(); let raw_msg = unsafe { DeserializeTestAllTypes((*data).as_ptr(), data.len()) }; - let msg2 = TestAllTypesView::__unstable_wrap_cpp_grant_permission_to_break(&raw_msg); + let msg2 = unsafe { TestAllTypesView::__unstable_wrap_raw_message(&raw_msg) }; proto_assert_eq!(msg1, msg2); @@ -130,14 +127,12 @@ fn deserialize_in_cpp_into_view() { // accidentally get destroyed by Rust. #[googletest::test] fn smuggle_extension() { - let msg1 = unsafe { - TestAllExtensions::__unstable_wrap_cpp_grant_permission_to_break(NewWithExtension()) - }; + let msg1 = + unsafe { TestAllExtensions::__unstable_take_ownership_of_raw_message(NewWithExtension()) }; let data = msg1.serialize().unwrap(); let mut msg2 = TestAllExtensions::parse(&data).unwrap(); - let bytes = unsafe { - GetBytesExtension(msg2.as_mut().__unstable_cpp_repr_grant_permission_to_break()).as_ref() - }; + let bytes = + unsafe { GetBytesExtension(msg2.as_mut().__unstable_as_raw_message_mut()).as_ref() }; assert_eq!(bytes, b"smuggled"); } diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 8f3cafaee1..d18a58deda 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -1287,8 +1287,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $nested_in_msg$ )rs"); + ctx.printer().PrintRaw("\n"); if (ctx.is_cpp()) { - ctx.printer().PrintRaw("\n"); ctx.Emit({{"Msg", RsSafeName(msg.name())}}, R"rs( impl $Msg$ { pub fn __unstable_wrap_cpp_grant_permission_to_break(msg: $pbr$::RawMessage) -> Self { @@ -1299,7 +1299,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { s.raw_msg() } } - impl<'a> $Msg$Mut<'a> { //~ msg is a &mut so that the borrow checker enforces exclusivity to //~ prevent constructing multiple Muts/Views from the same RawMessage. @@ -1325,9 +1324,58 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.msg } } + + impl $pb$::OwnedMessageInterop for $Msg$ { + unsafe fn __unstable_take_ownership_of_raw_message(msg: $pbr$::RawMessage) -> Self { + Self { inner: $pbr$::MessageInner { msg } } + } + + fn __unstable_leak_raw_message(self) -> $pbr$::RawMessage { + let s = std::mem::ManuallyDrop::new(self); + s.raw_msg() + } + } + + impl<'a> $pb$::MessageMutInterop<'a> for $Msg$Mut<'a> { + unsafe fn __unstable_wrap_raw_message_mut( + msg: &'a mut $pbr$::RawMessage) -> Self { + Self { + inner: $pbr$::MutatorMessageRef::from_raw_msg($pbi$::Private, msg) + } + } + fn __unstable_as_raw_message_mut(&mut self) -> $pbr$::RawMessage { + self.raw_msg() + } + } + + impl<'a> $pb$::MessageViewInterop<'a> for $Msg$View<'a> { + unsafe fn __unstable_wrap_raw_message( + msg: &'a $pbr$::RawMessage) -> Self { + Self::new($pbi$::Private, *msg) + } + fn __unstable_as_raw_message(&self) -> $pbr$::RawMessage { + self.msg + } + } + )rs"); + } else { + ctx.Emit({{"Msg", RsSafeName(msg.name())}}, R"rs( + // upb kernel doesn't support any owned message or message mut interop. + impl $pb$::OwnedMessageInterop for $Msg$ {} + impl<'a> $pb$::MessageMutInterop<'a> for $Msg$Mut<'a> {} + + impl<'a> $pb$::MessageViewInterop<'a> for $Msg$View<'a> { + unsafe fn __unstable_wrap_raw_message( + msg: &'a $pbr$::RawMessage) -> Self { + Self::new($pbi$::Private, *msg) + } + fn __unstable_as_raw_message(&self) -> $pbr$::RawMessage { + self.msg + } + } )rs"); } -} +} // NOLINT(readability/fn_size) // Generates code for a particular message in `.pb.thunk.cc`. void GenerateThunksCc(Context& ctx, const Descriptor& msg) { From 6ab302d3a3c0a09b2f0d0e951600e4b5b02686d0 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 30 Jul 2024 12:03:36 -0700 Subject: [PATCH 106/107] Rust: cut down on the amount of generated C++ code needed for maps With the C++ kernel for Rust, we currently need to generate quite a few C++ thunks for operations on map fields. For each message we generate, we generate these thunks for all possible map types that could have that message as a value. These operations are for things such as insertion, removal, clearing, iterating, etc. The reason we do this is that templated types don't play well with FFI, so we effectively need separate FFI endpoints for every possible combination of key and value types used (or even potentially used) as a map field. This CL fixes the problem by replacing the generated thunks with functions in the runtime that can operate on `proto2::MessageLite*` without needing to care about the specific message type. The way it works is that we implement the operations using either `UntypedMapBase` (the base class of all map types, which knows nothing about the key and value types) or `KeyMapBase`, which knows the key type but not the value type. I roughly followed the example of the table-driven parser, which has a similar problem of needing to operate generically on maps without having access to the concrete types. I removed 54 thunks per message (that's 6 key types times 9 operations per key), but had to add two new thunks per message: - The `size_info` thunk looks up the `MapNodeSizeInfoT`, which is stored in a small constant table. The important thing here is an offset indicating where to look for the value in each map entry. This offset can be different for every pair of key and value types, but we can safely assume that the result does not depend on the signedness of the key. As a result we only need to store four entries per message: one each for i32, i64, bool, and string. - The `placement_new` thunk move-constructs a message in place. We need this to be able to efficiently implement map insertion. There are two big things that this CL does not address yet but which I plan to follow up on: - Enums still generate many map-related C++ thunks that could be replaced with a common implementation. This should actually be much easier to handle than messages, because every enum has the same representation as an i32. - We still generate six `ProxiedInMapValue` implementations for every message, but it should be possible to replace these with a blanket implementation that works for all message types. PiperOrigin-RevId: 657681421 --- rust/cpp.rs | 104 ++++++++- rust/cpp_kernel/BUILD | 1 + rust/cpp_kernel/map.cc | 188 ++++++++++++++++ rust/cpp_kernel/map.h | 8 +- rust/test/shared/accessors_map_test.rs | 50 +++++ src/google/protobuf/compiler/rust/enum.cc | 3 +- src/google/protobuf/compiler/rust/message.cc | 224 ++++++++----------- src/google/protobuf/map.h | 66 ++++++ src/google/protobuf/message_lite.h | 2 + 9 files changed, 509 insertions(+), 137 deletions(-) diff --git a/rust/cpp.rs b/rust/cpp.rs index 4410f21d51..4d3358a8e8 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -95,6 +95,11 @@ pub struct InnerProtoString { owned_ptr: CppStdString, } +/// An opaque type matching MapNodeSizeInfoT from C++. +#[doc(hidden)] +#[repr(transparent)] +pub struct MapNodeSizeInfo(pub i32); + impl Drop for InnerProtoString { fn drop(&mut self) { // SAFETY: `self.owned_ptr` points to a valid std::string object. @@ -690,9 +695,11 @@ impl UntypedMapIterator { _private: Private, iter_get_thunk: unsafe extern "C" fn( iter: &mut UntypedMapIterator, + size_info: MapNodeSizeInfo, key: *mut FfiKey, value: *mut FfiValue, ), + size_info: MapNodeSizeInfo, from_ffi_key: impl FnOnce(FfiKey) -> View<'a, K>, from_ffi_value: impl FnOnce(FfiValue) -> View<'a, V>, ) -> Option<(View<'a, K>, View<'a, V>)> @@ -710,7 +717,7 @@ impl UntypedMapIterator { // - The iterator is not at the end (node is non-null). // - `ffi_key` and `ffi_value` are not read (as uninit) as promised by the // caller. - unsafe { (iter_get_thunk)(self, ffi_key.as_mut_ptr(), ffi_value.as_mut_ptr()) } + unsafe { (iter_get_thunk)(self, size_info, ffi_key.as_mut_ptr(), ffi_value.as_mut_ptr()) } // SAFETY: // - The backing map is alive as promised by the caller. @@ -733,8 +740,100 @@ impl UntypedMapIterator { } } +#[doc(hidden)] +#[repr(transparent)] +pub struct MapNodeSizeInfoIndex(i32); + +#[doc(hidden)] +pub trait MapNodeSizeInfoIndexForType { + const SIZE_INFO_INDEX: MapNodeSizeInfoIndex; +} + +macro_rules! generate_map_node_size_info_mapping { + ( $($key:ty, $index:expr;)* ) => { + $( + impl MapNodeSizeInfoIndexForType for $key { + const SIZE_INFO_INDEX: MapNodeSizeInfoIndex = MapNodeSizeInfoIndex($index); + } + )* + } +} + +// LINT.IfChange(size_info_mapping) +generate_map_node_size_info_mapping!( + i32, 0; + u32, 0; + i64, 1; + u64, 1; + bool, 2; + ProtoString, 3; +); +// LINT.ThenChange(//depot/google3/third_party/protobuf/compiler/rust/message. +// cc:size_info_mapping) + +macro_rules! impl_map_primitives { + (@impl $(($rust_type:ty, $cpp_type:ty) => [ + $insert_thunk:ident, + $get_thunk:ident, + $iter_get_thunk:ident, + $remove_thunk:ident, + ]),* $(,)?) => { + $( + extern "C" { + pub fn $insert_thunk( + m: RawMap, + size_info: MapNodeSizeInfo, + key: $cpp_type, + value: RawMessage, + placement_new: unsafe extern "C" fn(*mut c_void, m: RawMessage), + ) -> bool; + pub fn $get_thunk( + m: RawMap, + size_info: MapNodeSizeInfo, + key: $cpp_type, + value: *mut RawMessage, + ) -> bool; + pub fn $iter_get_thunk( + iter: &mut UntypedMapIterator, + size_info: MapNodeSizeInfo, + key: *mut $cpp_type, + value: *mut RawMessage, + ); + pub fn $remove_thunk(m: RawMap, size_info: MapNodeSizeInfo, key: $cpp_type) -> bool; + } + )* + }; + ($($rust_type:ty, $cpp_type:ty;)* $(,)?) => { + paste!{ + impl_map_primitives!(@impl $( + ($rust_type, $cpp_type) => [ + [< proto2_rust_map_insert_ $rust_type >], + [< proto2_rust_map_get_ $rust_type >], + [< proto2_rust_map_iter_get_ $rust_type >], + [< proto2_rust_map_remove_ $rust_type >], + ], + )*); + } + }; +} + +impl_map_primitives!( + i32, i32; + u32, u32; + i64, i64; + u64, u64; + bool, bool; + ProtoString, PtrAndLen; +); + extern "C" { fn proto2_rust_thunk_UntypedMapIterator_increment(iter: &mut UntypedMapIterator); + + pub fn proto2_rust_map_new() -> RawMap; + pub fn proto2_rust_map_free(m: RawMap, key_is_string: bool, size_info: MapNodeSizeInfo); + pub fn proto2_rust_map_clear(m: RawMap, key_is_string: bool, size_info: MapNodeSizeInfo); + pub fn proto2_rust_map_size(m: RawMap) -> usize; + pub fn proto2_rust_map_iter(m: RawMap) -> UntypedMapIterator; } macro_rules! impl_ProxiedInMapValue_for_non_generated_value_types { @@ -748,7 +847,7 @@ macro_rules! impl_ProxiedInMapValue_for_non_generated_value_types { fn [< proto2_rust_thunk_Map_ $key_t _ $t _insert >](m: RawMap, key: $ffi_key_t, value: $ffi_value_t) -> bool; fn [< proto2_rust_thunk_Map_ $key_t _ $t _get >](m: RawMap, key: $ffi_key_t, value: *mut $ffi_view_t) -> bool; fn [< proto2_rust_thunk_Map_ $key_t _ $t _iter >](m: RawMap) -> UntypedMapIterator; - fn [< proto2_rust_thunk_Map_ $key_t _ $t _iter_get >](iter: &mut UntypedMapIterator, key: *mut $ffi_key_t, value: *mut $ffi_view_t); + fn [< proto2_rust_thunk_Map_ $key_t _ $t _iter_get >](iter: &mut UntypedMapIterator, size_info: MapNodeSizeInfo, key: *mut $ffi_key_t, value: *mut $ffi_view_t); fn [< proto2_rust_thunk_Map_ $key_t _ $t _remove >](m: RawMap, key: $ffi_key_t, value: *mut $ffi_view_t) -> bool; } @@ -829,6 +928,7 @@ macro_rules! impl_ProxiedInMapValue_for_non_generated_value_types { iter.as_raw_mut(Private).next_unchecked::<$key_t, Self, _, _>( Private, [< proto2_rust_thunk_Map_ $key_t _ $t _iter_get >], + MapNodeSizeInfo(0), $from_ffi_key, $from_ffi_value, ) diff --git a/rust/cpp_kernel/BUILD b/rust/cpp_kernel/BUILD index 7d0a256f1c..c1c1cdb7e1 100644 --- a/rust/cpp_kernel/BUILD +++ b/rust/cpp_kernel/BUILD @@ -27,6 +27,7 @@ cc_library( "//src/google/protobuf:protobuf_lite", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/rust/cpp_kernel/map.cc b/rust/cpp_kernel/map.cc index 2418c7274c..60f53c7df0 100644 --- a/rust/cpp_kernel/map.cc +++ b/rust/cpp_kernel/map.cc @@ -1,12 +1,137 @@ #include "rust/cpp_kernel/map.h" +#include #include #include +#include #include #include "google/protobuf/map.h" +#include "google/protobuf/message_lite.h" #include "rust/cpp_kernel/strings.h" +namespace google { +namespace protobuf { +namespace rust { +namespace { + +template +struct FromViewType { + using type = T; +}; + +template <> +struct FromViewType { + using type = std::string; +}; + +template +using KeyMap = internal::KeyMapBase< + internal::KeyForBase::type>>; + +template +void DestroyMapNode(internal::UntypedMapBase* m, internal::NodeBase* node, + internal::MapNodeSizeInfoT size_info) { + if constexpr (std::is_same::value) { + static_cast(node->GetVoidKey())->~basic_string(); + } + internal::RustMapHelper::DestroyMessage( + static_cast(node->GetVoidValue(size_info))); + internal::RustMapHelper::DeallocNode(m, node, size_info); +} + +template +bool Insert(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + Key key, MessageLite* value, + void (*placement_new)(void*, MessageLite*)) { + internal::NodeBase* node = internal::RustMapHelper::AllocNode(m, size_info); + if constexpr (std::is_same::value) { + new (node->GetVoidKey()) std::string(key.ptr, key.len); + } else { + *static_cast(node->GetVoidKey()) = key; + } + void* value_ptr = node->GetVoidValue(size_info); + placement_new(value_ptr, value); + node = internal::RustMapHelper::InsertOrReplaceNode( + static_cast*>(m), node); + if (node == nullptr) { + return true; + } + DestroyMapNode(m, node, size_info); + return false; +} + +template ::value>::type> +internal::RustMapHelper::NodeAndBucket FindHelper(Map* m, Key key) { + return internal::RustMapHelper::FindHelper( + m, static_cast>(key)); +} + +template +internal::RustMapHelper::NodeAndBucket FindHelper(Map* m, + google::protobuf::rust::PtrAndLen key) { + return internal::RustMapHelper::FindHelper( + m, absl::string_view(key.ptr, key.len)); +} + +template +bool Get(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + Key key, MessageLite** value) { + auto* map_base = static_cast*>(m); + internal::RustMapHelper::NodeAndBucket result = FindHelper(map_base, key); + if (result.node == nullptr) { + return false; + } + *value = static_cast(result.node->GetVoidValue(size_info)); + return true; +} + +template +bool Remove(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + Key key) { + auto* map_base = static_cast*>(m); + internal::RustMapHelper::NodeAndBucket result = FindHelper(map_base, key); + if (result.node == nullptr) { + return false; + } + internal::RustMapHelper::EraseNoDestroy(map_base, result.bucket, result.node); + DestroyMapNode(m, result.node, size_info); + return true; +} + +template +void IterGet(const internal::UntypedMapIterator* iter, + internal::MapNodeSizeInfoT size_info, Key* key, + MessageLite** value) { + internal::NodeBase* node = iter->node_; + if constexpr (std::is_same::value) { + const std::string* s = static_cast(node->GetVoidKey()); + *key = PtrAndLen(s->data(), s->size()); + } else { + *key = *static_cast(node->GetVoidKey()); + } + *value = static_cast(node->GetVoidValue(size_info)); +} + +void ClearMap(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + bool key_is_string, bool reset_table) { + if (internal::RustMapHelper::IsGlobalEmptyTable(m)) return; + uint8_t bits = internal::RustMapHelper::kValueIsProto; + if (key_is_string) { + bits |= internal::RustMapHelper::kKeyIsString; + } + internal::RustMapHelper::ClearTable( + m, internal::RustMapHelper::ClearInput{size_info, bits, reset_table, + /* destroy_node = */ nullptr}); +} + +} // namespace +} // namespace rust +} // namespace protobuf +} // namespace google + extern "C" { void proto2_rust_thunk_UntypedMapIterator_increment( @@ -14,6 +139,69 @@ void proto2_rust_thunk_UntypedMapIterator_increment( iter->PlusPlus(); } +google::protobuf::internal::UntypedMapBase* proto2_rust_map_new() { + return new google::protobuf::internal::UntypedMapBase(/* arena = */ nullptr); +} + +void proto2_rust_map_free(google::protobuf::internal::UntypedMapBase* m, + bool key_is_string, + google::protobuf::internal::MapNodeSizeInfoT size_info) { + google::protobuf::rust::ClearMap(m, size_info, key_is_string, + /* reset_table = */ false); + delete m; +} + +void proto2_rust_map_clear(google::protobuf::internal::UntypedMapBase* m, + bool key_is_string, + google::protobuf::internal::MapNodeSizeInfoT size_info) { + google::protobuf::rust::ClearMap(m, size_info, key_is_string, /* reset_table = */ true); +} + +size_t proto2_rust_map_size(google::protobuf::internal::UntypedMapBase* m) { + return m->size(); +} + +google::protobuf::internal::UntypedMapIterator proto2_rust_map_iter( + google::protobuf::internal::UntypedMapBase* m) { + return m->begin(); +} + +#define DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(cpp_type, suffix) \ + bool proto2_rust_map_insert_##suffix( \ + google::protobuf::internal::UntypedMapBase* m, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type key, \ + google::protobuf::MessageLite* value, \ + void (*placement_new)(void*, google::protobuf::MessageLite*)) { \ + return google::protobuf::rust::Insert(m, size_info, key, value, placement_new); \ + } \ + \ + bool proto2_rust_map_get_##suffix( \ + google::protobuf::internal::UntypedMapBase* m, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type key, \ + google::protobuf::MessageLite** value) { \ + return google::protobuf::rust::Get(m, size_info, key, value); \ + } \ + \ + bool proto2_rust_map_remove_##suffix( \ + google::protobuf::internal::UntypedMapBase* m, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type key) { \ + return google::protobuf::rust::Remove(m, size_info, key); \ + } \ + \ + void proto2_rust_map_iter_get_##suffix( \ + const google::protobuf::internal::UntypedMapIterator* iter, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type* key, \ + google::protobuf::MessageLite** value) { \ + return google::protobuf::rust::IterGet(iter, size_info, key, value); \ + } + +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(int32_t, i32) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(uint32_t, u32) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(int64_t, i64) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(uint64_t, u64) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(bool, bool) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(google::protobuf::rust::PtrAndLen, ProtoString) + __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE(int32_t, i32, int32_t, int32_t, value, cpp_value); __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE(uint32_t, u32, uint32_t, diff --git a/rust/cpp_kernel/map.h b/rust/cpp_kernel/map.h index c16f14b1f3..e568222edb 100644 --- a/rust/cpp_kernel/map.h +++ b/rust/cpp_kernel/map.h @@ -4,6 +4,10 @@ #include #include +#include "google/protobuf/map.h" +#include "google/protobuf/message_lite.h" +#include "rust/cpp_kernel/strings.h" + namespace google { namespace protobuf { namespace rust { @@ -74,8 +78,8 @@ auto MakeCleanup(T value) { return google::protobuf::internal::UntypedMapIterator::FromTyped(m->cbegin()); \ } \ void proto2_rust_thunk_Map_##rust_key_ty##_##rust_value_ty##_iter_get( \ - const google::protobuf::internal::UntypedMapIterator* iter, ffi_key_ty* key, \ - ffi_view_ty* value) { \ + const google::protobuf::internal::UntypedMapIterator* iter, int32_t, \ + ffi_key_ty* key, ffi_view_ty* value) { \ auto typed_iter = \ iter->ToTyped::const_iterator>(); \ const auto& cpp_key = typed_iter->first; \ diff --git a/rust/test/shared/accessors_map_test.rs b/rust/test/shared/accessors_map_test.rs index 4ec026eb1f..942efb6efd 100644 --- a/rust/test/shared/accessors_map_test.rs +++ b/rust/test/shared/accessors_map_test.rs @@ -212,6 +212,56 @@ fn test_map_setter() { } } +#[test] +fn test_map_creation_with_message_values() { + // Maps are usually created and owned by a parent message, but let's verify that + // we can successfully create and destroy them independently. + macro_rules! test_for_each_key { + ($($key_t:ty, $key:expr;)*) => { + $( + let msg = TestAllTypes::new(); + let mut map = protobuf::Map::<$key_t, TestAllTypes>::new(); + map.as_mut().insert($key, msg); + assert_that!(map.as_view().len(), eq(1)); + )* + } + } + + test_for_each_key!( + i32, -5; + u32, 13u32; + i64, 7; + u64, 11u64; + bool, false; + ProtoString, "looooooooooooooooooooooooong string"; + ); +} + +#[test] +fn test_map_clearing_with_message_values() { + macro_rules! test_for_each_key { + ($($key_t:ty, $key:expr;)*) => { + $( + let msg = TestAllTypes::new(); + let mut map = protobuf::Map::<$key_t, TestAllTypes>::new(); + map.as_mut().insert($key, msg); + assert_that!(map.as_view().len(), eq(1)); + map.as_mut().clear(); + assert_that!(map.as_view().len(), eq(0)); + )* + } + } + + test_for_each_key!( + i32, -5; + u32, 13u32; + i64, 7; + u64, 11u64; + bool, false; + ProtoString, "looooooooooooooooooooooooong string"; + ); +} + macro_rules! generate_map_with_msg_values_tests { ( $(($k_field:ident, $k_nonzero:expr, $k_other:expr $(,)?)),* diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 9b4a4402cf..8d5ccb54e9 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -81,7 +81,7 @@ void EnumProxiedInMapValue(Context& ctx, const EnumDescriptor& desc) { fn $map_get_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $name$) -> bool; fn $map_remove_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $name$) -> bool; fn $map_iter_thunk$(m: $pbr$::RawMap) -> $pbr$::UntypedMapIterator; - fn $map_iter_get_thunk$(iter: &mut $pbr$::UntypedMapIterator, key: *mut $ffi_key_t$, value: *mut $name$); + fn $map_iter_get_thunk$(iter: &mut $pbr$::UntypedMapIterator, size_info: $pbr$::MapNodeSizeInfo, key: *mut $ffi_key_t$, value: *mut $name$); } impl $pb$::ProxiedInMapValue<$key_t$> for $name$ { fn map_new(_private: $pbi$::Private) -> $pb$::Map<$key_t$, Self> { @@ -149,6 +149,7 @@ void EnumProxiedInMapValue(Context& ctx, const EnumDescriptor& desc) { iter.as_raw_mut($pbi$::Private).next_unchecked::<$key_t$, Self, _, _>( $pbi$::Private, $map_iter_get_thunk$, + $pbr$::MapNodeSizeInfo(0), // Ignored |ffi_key| $from_ffi_key_expr$, $std$::convert::identity, ) diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index d18a58deda..4665f44bf6 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -11,6 +11,7 @@ #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/cpp/names.h" @@ -197,29 +198,28 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: ctx.Emit( - { - {"new_thunk", ThunkName(ctx, msg, "new")}, - {"delete_thunk", ThunkName(ctx, msg, "delete")}, - {"clear_thunk", ThunkName(ctx, msg, "clear")}, - {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, - {"parse_thunk", ThunkName(ctx, msg, "parse")}, - {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, - {"merge_from_thunk", ThunkName(ctx, msg, "merge_from")}, - {"repeated_new_thunk", ThunkName(ctx, msg, "repeated_new")}, - {"repeated_free_thunk", ThunkName(ctx, msg, "repeated_free")}, - {"repeated_len_thunk", ThunkName(ctx, msg, "repeated_len")}, - {"repeated_get_thunk", ThunkName(ctx, msg, "repeated_get")}, - {"repeated_get_mut_thunk", - ThunkName(ctx, msg, "repeated_get_mut")}, - {"repeated_add_thunk", ThunkName(ctx, msg, "repeated_add")}, - {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, - {"repeated_copy_from_thunk", - ThunkName(ctx, msg, "repeated_copy_from")}, - {"repeated_reserve_thunk", - ThunkName(ctx, msg, "repeated_reserve")}, - }, + {{"new_thunk", ThunkName(ctx, msg, "new")}, + {"placement_new_thunk", ThunkName(ctx, msg, "placement_new")}, + {"delete_thunk", ThunkName(ctx, msg, "delete")}, + {"clear_thunk", ThunkName(ctx, msg, "clear")}, + {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, + {"parse_thunk", ThunkName(ctx, msg, "parse")}, + {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, + {"merge_from_thunk", ThunkName(ctx, msg, "merge_from")}, + {"repeated_new_thunk", ThunkName(ctx, msg, "repeated_new")}, + {"repeated_free_thunk", ThunkName(ctx, msg, "repeated_free")}, + {"repeated_len_thunk", ThunkName(ctx, msg, "repeated_len")}, + {"repeated_get_thunk", ThunkName(ctx, msg, "repeated_get")}, + {"repeated_get_mut_thunk", ThunkName(ctx, msg, "repeated_get_mut")}, + {"repeated_add_thunk", ThunkName(ctx, msg, "repeated_add")}, + {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, + {"repeated_copy_from_thunk", + ThunkName(ctx, msg, "repeated_copy_from")}, + {"repeated_reserve_thunk", ThunkName(ctx, msg, "repeated_reserve")}, + {"map_size_info_thunk", ThunkName(ctx, msg, "size_info")}}, R"rs( fn $new_thunk$() -> $pbr$::RawMessage; + fn $placement_new_thunk$(ptr: *mut std::ffi::c_void, m: $pbr$::RawMessage); fn $delete_thunk$(raw_msg: $pbr$::RawMessage); fn $clear_thunk$(raw_msg: $pbr$::RawMessage); fn $serialize_thunk$(raw_msg: $pbr$::RawMessage, out: &mut $pbr$::SerializedData) -> bool; @@ -235,6 +235,7 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { fn $repeated_clear_thunk$(raw: $pbr$::RawRepeatedField); fn $repeated_copy_from_thunk$(dst: $pbr$::RawRepeatedField, src: $pbr$::RawRepeatedField); fn $repeated_reserve_thunk$(raw: $pbr$::RawRepeatedField, additional: usize); + fn $map_size_info_thunk$(i: $pbr$::MapNodeSizeInfoIndex) -> $pbr$::MapNodeSizeInfo; )rs"); return; @@ -590,19 +591,18 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { case Kernel::kCpp: for (const auto& t : kMapKeyTypes) { ctx.Emit( - {{"map_new_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "new")}, - {"map_free_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "free")}, - {"map_clear_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "clear")}, - {"map_size_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "size")}, - {"map_insert_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "insert")}, - {"map_get_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "get")}, - {"map_remove_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "remove")}, - {"map_iter_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "iter")}, - {"map_iter_get_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "iter_get")}, + {{"map_size_info_thunk", ThunkName(ctx, msg, "size_info")}, + {"placement_new_thunk", ThunkName(ctx, msg, "placement_new")}, + {"map_insert", + absl::StrCat("proto2_rust_map_insert_", t.thunk_ident)}, + {"map_remove", + absl::StrCat("proto2_rust_map_remove_", t.thunk_ident)}, + {"map_get", absl::StrCat("proto2_rust_map_get_", t.thunk_ident)}, + {"map_iter_get", + absl::StrCat("proto2_rust_map_iter_get_", t.thunk_ident)}, {"key_expr", t.rs_to_ffi_key_expr}, + {"key_is_string", + t.thunk_ident == "ProtoString" ? "true" : "false"}, io::Printer::Sub("ffi_key_t", [&] { ctx.Emit(t.rs_ffi_key_t); }) .WithSuffix(""), io::Printer::Sub("key_t", [&] { ctx.Emit(t.rs_key_t); }) @@ -611,47 +611,52 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { [&] { ctx.Emit(t.rs_from_ffi_key_expr); }) .WithSuffix("")}, R"rs( - extern "C" { - fn $map_new_thunk$() -> $pbr$::RawMap; - fn $map_free_thunk$(m: $pbr$::RawMap); - fn $map_clear_thunk$(m: $pbr$::RawMap); - fn $map_size_thunk$(m: $pbr$::RawMap) -> usize; - fn $map_insert_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: $pbr$::RawMessage) -> bool; - fn $map_get_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $pbr$::RawMessage) -> bool; - fn $map_remove_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $pbr$::RawMessage) -> bool; - fn $map_iter_thunk$(m: $pbr$::RawMap) -> $pbr$::UntypedMapIterator; - fn $map_iter_get_thunk$(iter: &mut $pbr$::UntypedMapIterator, key: *mut $ffi_key_t$, value: *mut $pbr$::RawMessage); - } impl $pb$::ProxiedInMapValue<$key_t$> for $Msg$ { fn map_new(_private: $pbi$::Private) -> $pb$::Map<$key_t$, Self> { unsafe { $pb$::Map::from_inner( $pbi$::Private, - $pbr$::InnerMap::new($pbi$::Private, $map_new_thunk$()) + $pbr$::InnerMap::new($pbi$::Private, $pbr$::proto2_rust_map_new()) ) } } unsafe fn map_free(_private: $pbi$::Private, map: &mut $pb$::Map<$key_t$, Self>) { - unsafe { $map_free_thunk$(map.as_raw($pbi$::Private)); } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { $pbr$::proto2_rust_map_free(map.as_raw($pbi$::Private), $key_is_string$, $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX)); } } fn map_clear(mut map: $pb$::Mut<'_, $pb$::Map<$key_t$, Self>>) { - unsafe { $map_clear_thunk$(map.as_raw($pbi$::Private)); } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { $pbr$::proto2_rust_map_clear(map.as_raw($pbi$::Private), $key_is_string$, $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX)); } } fn map_len(map: $pb$::View<'_, $pb$::Map<$key_t$, Self>>) -> usize { - unsafe { $map_size_thunk$(map.as_raw($pbi$::Private)) } + unsafe { $pbr$::proto2_rust_map_size(map.as_raw($pbi$::Private)) } } fn map_insert(mut map: $pb$::Mut<'_, $pb$::Map<$key_t$, Self>>, key: $pb$::View<'_, $key_t$>, value: impl $pb$::IntoProxied) -> bool { - unsafe { $map_insert_thunk$(map.as_raw($pbi$::Private), $key_expr$, value.into_proxied($pbi$::Private).raw_msg()) } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { + $pbr$::$map_insert$( + map.as_raw($pbi$::Private), + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), + $key_expr$, + value.into_proxied($pbi$::Private).raw_msg(), $placement_new_thunk$) + } } fn map_get<'a>(map: $pb$::View<'a, $pb$::Map<$key_t$, Self>>, key: $pb$::View<'_, $key_t$>) -> Option<$pb$::View<'a, Self>> { + use $pbr$::MapNodeSizeInfoIndexForType; let key = $key_expr$; let mut value = $std$::mem::MaybeUninit::uninit(); - let found = unsafe { $map_get_thunk$(map.as_raw($pbi$::Private), key, value.as_mut_ptr()) }; + let found = unsafe { + $pbr$::$map_get$( + map.as_raw($pbi$::Private), + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), + key, + value.as_mut_ptr()) + }; if !found { return None; } @@ -659,8 +664,13 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { } fn map_remove(mut map: $pb$::Mut<'_, $pb$::Map<$key_t$, Self>>, key: $pb$::View<'_, $key_t$>) -> bool { - let mut value = $std$::mem::MaybeUninit::uninit(); - unsafe { $map_remove_thunk$(map.as_raw($pbi$::Private), $key_expr$, value.as_mut_ptr()) } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { + $pbr$::$map_remove$( + map.as_raw($pbi$::Private), + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), + $key_expr$) + } } fn map_iter(map: $pb$::View<'_, $pb$::Map<$key_t$, Self>>) -> $pb$::MapIter<'_, $key_t$, Self> { @@ -672,12 +682,13 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { unsafe { $pb$::MapIter::from_raw( $pbi$::Private, - $map_iter_thunk$(map.as_raw($pbi$::Private)) + $pbr$::proto2_rust_map_iter(map.as_raw($pbi$::Private)) ) } } fn map_iter_next<'a>(iter: &mut $pb$::MapIter<'a, $key_t$, Self>) -> Option<($pb$::View<'a, $key_t$>, $pb$::View<'a, Self>)> { + use $pbr$::MapNodeSizeInfoIndexForType; // SAFETY: // - The `MapIter` API forbids the backing map from being mutated for 'a, // and guarantees that it's the correct key and value types. @@ -687,7 +698,8 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { unsafe { iter.as_raw_mut($pbi$::Private).next_unchecked::<$key_t$, Self, _, _>( $pbi$::Private, - $map_iter_get_thunk$, + $pbr$::$map_iter_get$, + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), |ffi_key| $from_ffi_key_expr$, |raw_msg| $Msg$View::new($pbi$::Private, raw_msg) ) @@ -1390,6 +1402,7 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { {"Msg", RsSafeName(msg.name())}, {"QualifiedMsg", cpp::QualifiedClassName(&msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, + {"placement_new_thunk", ThunkName(ctx, msg, "placement_new")}, {"delete_thunk", ThunkName(ctx, msg, "delete")}, {"clear_thunk", ThunkName(ctx, msg, "clear")}, {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, @@ -1405,6 +1418,7 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, {"repeated_copy_from_thunk", ThunkName(ctx, msg, "repeated_copy_from")}, {"repeated_reserve_thunk", ThunkName(ctx, msg, "repeated_reserve")}, + {"map_size_info_thunk", ThunkName(ctx, msg, "size_info")}, {"nested_msg_thunks", [&] { for (int i = 0; i < msg.nested_type_count(); ++i) { @@ -1427,12 +1441,15 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { } }}}, R"cc( - //~ $abi$ is a workaround for a syntax highlight bug in VSCode. However, - //~ that confuses clang-format (it refuses to keep the newline after - //~ `$abi${`). Disabling clang-format for the block. + //~ $abi$ is a workaround for a syntax highlight bug in VSCode. + // However, ~ that confuses clang-format (it refuses to keep the + // newline after ~ `$abi${`). Disabling clang-format for the block. // clang-format off extern $abi$ { void* $new_thunk$() { return new $QualifiedMsg$(); } + void $placement_new_thunk$(void* ptr, $QualifiedMsg$& m) { + new (ptr) $QualifiedMsg$(std::move(m)); + } void $delete_thunk$(void* ptr) { delete static_cast<$QualifiedMsg$*>(ptr); } void $clear_thunk$(void* ptr) { static_cast<$QualifiedMsg$*>(ptr)->Clear(); @@ -1490,89 +1507,32 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { size_t additional) { field->Reserve(field->size() + additional); } + google::protobuf::internal::MapNodeSizeInfoT $map_size_info_thunk$(int32_t i) { + static constexpr google::protobuf::internal::MapNodeSizeInfoT size_infos[] = {)cc" + // LINT.IfChange(size_info_mapping) + R"cc( + google::protobuf::internal::RustMapHelper::SizeInfo(), + google::protobuf::internal::RustMapHelper::SizeInfo(), + google::protobuf::internal::RustMapHelper::SizeInfo(), + google::protobuf::internal::RustMapHelper::SizeInfo() + )cc" + // LINT.ThenChange(//depot/google3/third_party/protobuf/rust/cpp.rs:size_info_mapping) + R"cc( + } + ; + return size_infos[i]; + } $accessor_thunks$ - $oneof_thunks$ + $oneof_thunks$ } // extern $abi$ // clang-format on $nested_msg_thunks$ )cc"); - for (const auto& t : kMapKeyTypes) { - ctx.Emit( - { - {"map_new_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "new")}, - {"map_free_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "free")}, - {"map_clear_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "clear")}, - {"map_size_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "size")}, - {"map_insert_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "insert")}, - {"map_get_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "get")}, - {"map_remove_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "remove")}, - {"map_iter_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "iter")}, - {"map_iter_get_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "iter_get")}, - {"key_t", t.cc_key_t}, - {"ffi_key_t", t.cc_ffi_key_t}, - {"key_expr", t.cc_from_ffi_key_expr}, - {"to_ffi_key_expr", t.cc_to_ffi_key_expr}, - {"pkg::Msg", cpp::QualifiedClassName(&msg)}, - {"abi", "\"C\""}, // Workaround for syntax highlight bug in VSCode. - }, - R"cc( - extern $abi$ { - const google::protobuf::Map<$key_t$, $pkg::Msg$>* $map_new_thunk$() { - return new google::protobuf::Map<$key_t$, $pkg::Msg$>(); - } - void $map_free_thunk$(const google::protobuf::Map<$key_t$, $pkg::Msg$>* m) { delete m; } - void $map_clear_thunk$(google::protobuf::Map<$key_t$, $pkg::Msg$> * m) { m->clear(); } - size_t $map_size_thunk$(const google::protobuf::Map<$key_t$, $pkg::Msg$>* m) { - return m->size(); - } - bool $map_insert_thunk$(google::protobuf::Map<$key_t$, $pkg::Msg$> * m, - $ffi_key_t$ key, $pkg::Msg$ value) { - auto k = $key_expr$; - auto it = m->find(k); - if (it != m->end()) { - return false; - } - (*m)[k] = value; - return true; - } - bool $map_get_thunk$(const google::protobuf::Map<$key_t$, $pkg::Msg$>* m, - $ffi_key_t$ key, const $pkg::Msg$** value) { - auto it = m->find($key_expr$); - if (it == m->end()) { - return false; - } - const $pkg::Msg$* cpp_value = &it->second; - *value = cpp_value; - return true; - } - bool $map_remove_thunk$(google::protobuf::Map<$key_t$, $pkg::Msg$> * m, - $ffi_key_t$ key, $pkg::Msg$ * value) { - auto num_removed = m->erase($key_expr$); - return num_removed > 0; - } - google::protobuf::internal::UntypedMapIterator $map_iter_thunk$( - const google::protobuf::Map<$key_t$, $pkg::Msg$>* m) { - return google::protobuf::internal::UntypedMapIterator::FromTyped(m->cbegin()); - } - void $map_iter_get_thunk$( - const google::protobuf::internal::UntypedMapIterator* iter, - $ffi_key_t$* key, const $pkg::Msg$** value) { - auto typed_iter = iter->ToTyped< - google::protobuf::Map<$key_t$, $pkg::Msg$>::const_iterator>(); - const auto& cpp_key = typed_iter->first; - const auto& cpp_value = typed_iter->second; - *key = $to_ffi_key_expr$; - *value = &cpp_value; - } - } - )cc"); - } } } // namespace rust diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 0953b12d00..74cfa9d439 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -62,6 +62,10 @@ class MapIterator; template struct is_proto_enum; +namespace rust { +struct PtrAndLen; +} // namespace rust + namespace internal { template class MapFieldLite; @@ -590,6 +594,7 @@ class PROTOBUF_EXPORT UntypedMapBase { friend struct MapTestPeer; friend struct MapBenchmarkPeer; friend class UntypedMapIterator; + friend class RustMapHelper; struct NodeAndBucket { NodeBase* node; @@ -924,6 +929,7 @@ class KeyMapBase : public UntypedMapBase { friend class TcParser; friend struct MapTestPeer; friend struct MapBenchmarkPeer; + friend class RustMapHelper; PROTOBUF_NOINLINE void erase_no_destroy(map_index_t b, KeyNode* node) { TreeIterator tree_it; @@ -1139,6 +1145,60 @@ bool InitializeMapKey(T*, K&&, Arena*) { } +// The purpose of this class is to give the Rust implementation visibility into +// some of the internals of C++ proto maps. We need access to these internals +// to be able to implement Rust map operations without duplicating the same +// functionality for every message type. +class RustMapHelper { + public: + using NodeAndBucket = UntypedMapBase::NodeAndBucket; + using ClearInput = UntypedMapBase::ClearInput; + + template + static constexpr MapNodeSizeInfoT SizeInfo() { + return Map::Node::size_info(); + } + + enum { + kKeyIsString = UntypedMapBase::kKeyIsString, + kValueIsProto = UntypedMapBase::kValueIsProto, + }; + + static NodeBase* AllocNode(UntypedMapBase* m, MapNodeSizeInfoT size_info) { + return m->AllocNode(size_info); + } + + static void DeallocNode(UntypedMapBase* m, NodeBase* node, + MapNodeSizeInfoT size_info) { + return m->DeallocNode(node, size_info); + } + + template + static NodeAndBucket FindHelper(Map* m, Key key) { + return m->FindHelper(key); + } + + template + static typename Map::KeyNode* InsertOrReplaceNode(Map* m, NodeBase* node) { + return m->InsertOrReplaceNode(static_cast(node)); + } + + template + static void EraseNoDestroy(Map* m, map_index_t bucket, NodeBase* node) { + m->erase_no_destroy(bucket, static_cast(node)); + } + + static void DestroyMessage(MessageLite* m) { m->DestroyInstance(false); } + + static void ClearTable(UntypedMapBase* m, ClearInput input) { + m->ClearTable(input); + } + + static bool IsGlobalEmptyTable(const UntypedMapBase* m) { + return m->num_buckets_ == kGlobalEmptyTableSize; + } +}; + } // namespace internal // This is the class for Map's internal value_type. @@ -1252,6 +1312,11 @@ class Map : private internal::KeyMapBase> { internal::is_internal_map_value_type>::value, "We only support scalar, Message, and designated internal " "mapped types."); + // The Rust implementation that wraps C++ protos relies on the ability to + // create an UntypedMapBase and cast a pointer of it to google::protobuf::Map*. + static_assert( + sizeof(Map) == sizeof(internal::UntypedMapBase), + "Map must not have any data members beyond what is in UntypedMapBase."); } template @@ -1702,6 +1767,7 @@ class Map : private internal::KeyMapBase> { friend class internal::TcParser; friend struct internal::MapTestPeer; friend struct internal::MapBenchmarkPeer; + friend class internal::RustMapHelper; }; namespace internal { diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index 9e462fa5f9..94e18c1284 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -158,6 +158,7 @@ class TcParser; struct TcParseTableBase; class WireFormatLite; class WeakFieldMap; +class RustMapHelper; template class GenericTypeHandler; // defined in repeated_field.h @@ -857,6 +858,7 @@ class PROTOBUF_EXPORT MessageLite { friend class internal::UntypedMapBase; friend class internal::WeakFieldMap; friend class internal::WireFormatLite; + friend class internal::RustMapHelper; template friend class Arena::InternalHelper; From 06a520cdb78be66f506b88a8adba2763f2301dc0 Mon Sep 17 00:00:00 2001 From: Tony Liao Date: Tue, 30 Jul 2024 14:09:12 -0700 Subject: [PATCH 107/107] Add wire format parsing unit tests for implicit presence fields. This gist of the new test case coverage can be summarized as: - The wire does not distinguish between explicit and implicit fields. - For implicit presence fields, zeros on the wire can overwrite nonzero values (i.e. they are treated as a 'clear' operation). It's TBD whether we want to accept this behaviour going forward. Right now we are leaning towards keeping this behaviour, because: - If we receive zeros on the wire for implicit-presence fields, the protobuf wire format is "wrong" anyway. Well-behaved code should never generate zeros on the wire for implicit presence fields or serialize the same field multiple times. - There might be some value to enforce that "anything on the wire is accepted". This can make handling of wire format simpler. There are some drawbacks with this approach: - It might be somewhat surprising for users that zeros on the wire are always read, even for implicit-presence fields. - It might make the transition from implicit-presence to explicit-presence harder (or more unsafe) if user wants to migrate. - It leads to an inconsistency between what it means to "Merge". - Merging from a constructed object, with implicit presence and with field set to zero, will not overwrite. - Merging from the wire, with implicit presence and with zero explicitly present on the wire, WILL overwrite. I still need to add conformance tests to ensure that this is a consistent behavior across all languages, but for now let's at least add some coverage in C++ to ensure that this is a tested behaviour. PiperOrigin-RevId: 657724599 --- src/google/protobuf/BUILD.bazel | 2 + src/google/protobuf/no_field_presence_test.cc | 73 +++++++++++++++++++ .../protobuf/unittest_no_field_presence.proto | 8 +- 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 0321694748..82a2dc6ea6 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -1657,6 +1657,8 @@ cc_test( deps = [ ":cc_test_protos", ":protobuf", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings:string_view", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/google/protobuf/no_field_presence_test.cc b/src/google/protobuf/no_field_presence_test.cc index 56586731c7..a5192ca502 100644 --- a/src/google/protobuf/no_field_presence_test.cc +++ b/src/google/protobuf/no_field_presence_test.cc @@ -8,7 +8,10 @@ #include #include "google/protobuf/descriptor.pb.h" +#include #include +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/unittest.pb.h" #include "google/protobuf/unittest_no_field_presence.pb.h" @@ -17,6 +20,8 @@ namespace google { namespace protobuf { namespace { +using ::testing::StrEq; + // Helper: checks that all fields have default (zero/empty) values. void CheckDefaultValues( const proto2_nofieldpresence_unittest::TestAllTypes& m) { @@ -457,6 +462,74 @@ TEST(NoFieldPresenceTest, MergeFromIfNonzeroTest) { EXPECT_EQ("test2", dest.optional_string()); } +TEST(NoFieldPresenceTest, ExtraZeroesInWireParseTest) { + // check extra serialized zeroes on the wire are parsed into the object. + proto2_nofieldpresence_unittest::ForeignMessage dest; + dest.set_c(42); + ASSERT_EQ(42, dest.c()); + + // ExplicitForeignMessage has the same fields as ForeignMessage, but with + // explicit presence instead of implicit presence. + proto2_nofieldpresence_unittest::ExplicitForeignMessage source; + source.set_c(0); + std::string wire = source.SerializeAsString(); + ASSERT_THAT(wire, StrEq(absl::string_view{"\x08\x00", 2})); + + // The "parse" operation clears all fields before merging from wire. + ASSERT_TRUE(dest.ParseFromString(wire)); + EXPECT_EQ(0, dest.c()); + std::string dest_data; + EXPECT_TRUE(dest.SerializeToString(&dest_data)); + EXPECT_TRUE(dest_data.empty()); +} + +TEST(NoFieldPresenceTest, ExtraZeroesInWireMergeTest) { + // check explicit zeros on the wire are merged into an implicit one. + proto2_nofieldpresence_unittest::ForeignMessage dest; + dest.set_c(42); + ASSERT_EQ(42, dest.c()); + + // ExplicitForeignMessage has the same fields as ForeignMessage, but with + // explicit presence instead of implicit presence. + proto2_nofieldpresence_unittest::ExplicitForeignMessage source; + source.set_c(0); + std::string wire = source.SerializeAsString(); + ASSERT_THAT(wire, StrEq(absl::string_view{"\x08\x00", 2})); + + // TODO: b/356132170 -- Add conformance tests to ensure this behaviour is + // well-defined. + // As implemented, the C++ "merge" operation does not distinguish between + // implicit and explicit fields when reading from the wire. + ASSERT_TRUE(dest.MergeFromString(wire)); + // If zero is present on the wire, the original value is overwritten, even + // though this is specified as an "implicit presence" field. + EXPECT_EQ(0, dest.c()); + std::string dest_data; + EXPECT_TRUE(dest.SerializeToString(&dest_data)); + EXPECT_TRUE(dest_data.empty()); +} + +TEST(NoFieldPresenceTest, ExtraZeroesInWireLastWins) { + // check that, when the same field is present multiple times on the wire, we + // always take the last one -- even if it is a zero. + + absl::string_view wire{"\x08\x01\x08\x00", /*len=*/4}; // note the null-byte. + proto2_nofieldpresence_unittest::ForeignMessage dest; + + // TODO: b/356132170 -- Add conformance tests to ensure this behaviour is + // well-defined. + // As implemented, the C++ "merge" operation does not distinguish between + // implicit and explicit fields when reading from the wire. + ASSERT_TRUE(dest.MergeFromString(wire)); + // If the same field is present multiple times on the wire, "last one wins". + // i.e. -- the last seen field content will always overwrite, even if it's + // zero and the field is implicit presence. + EXPECT_EQ(0, dest.c()); + std::string dest_data; + EXPECT_TRUE(dest.SerializeToString(&dest_data)); + EXPECT_TRUE(dest_data.empty()); +} + TEST(NoFieldPresenceTest, IsInitializedTest) { // Check that IsInitialized works properly. proto2_nofieldpresence_unittest::TestProto2Required message; diff --git a/src/google/protobuf/unittest_no_field_presence.proto b/src/google/protobuf/unittest_no_field_presence.proto index f6880a58e9..cc02acc913 100644 --- a/src/google/protobuf/unittest_no_field_presence.proto +++ b/src/google/protobuf/unittest_no_field_presence.proto @@ -92,7 +92,7 @@ message TestAllTypes { repeated string repeated_string_piece = 54 [ctype = STRING_PIECE]; repeated string repeated_cord = 55 [ctype = CORD]; - repeated NestedMessage repeated_lazy_message = 57 ; + repeated NestedMessage repeated_lazy_message = 57; oneof oneof_field { uint32 oneof_uint32 = 111; @@ -112,6 +112,12 @@ message ForeignMessage { int32 c = 1; } +// Same as ForeignMessage, but all fields have explicit presence. +// It can be useful for testing explicit-implicit presence interop behaviour. +message ExplicitForeignMessage { + int32 c = 1 [features.field_presence = EXPLICIT]; +} + enum ForeignEnum { FOREIGN_FOO = 0; FOREIGN_BAR = 1;