Implement the lazy repeated field data structure.

PiperOrigin-RevId: 653379333

src/google/protobuf/lazy_repeated_field.cc

@@ -0,0 +1,344 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2023 Google Inc.  All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#include "google/protobuf/lazy_repeated_field.h"
+
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <string>
+#include <utility>
+
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
+#include "absl/log/log.h"
+#include "absl/strings/cord.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "google/protobuf/arena.h"
+#include "google/protobuf/generated_message_util.h"
+#include "google/protobuf/io/coded_stream.h"
+#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
+#include "google/protobuf/message_lite.h"
+#include "google/protobuf/parse_context.h"
+
+// Must be included last.
+// clang-format off
+#include "google/protobuf/port_def.inc"
+#include "google/protobuf/repeated_ptr_field.h"
+// clang-format on
+
+namespace google {
+namespace protobuf {
+namespace internal {
+namespace {}  // namespace
+
+namespace {
+
+inline const char* InternalParseRepeated(const char* ptr,
+                                         ParseContext* local_ctx,
+                                         RepeatedPtrFieldBase* value,
+                                         const MessageLite* prototype) {
+  uint32_t expected_tag;
+  ptr = ReadTag(ptr, &expected_tag);
+  if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr;
+  // TODO: Try to optimize this. The tags and lengths are read again
+  // which is a bit wasteful.
+  return LazyRepeatedPtrField::ParseToRepeatedMessage<uint32_t>(
+      ptr, local_ctx, prototype, expected_tag, value);
+}
+
+template <typename T>
+inline bool ParseWithNullOuterContextImpl(const T& input,
+                                          RepeatedPtrFieldBase* value,
+                                          const MessageLite* prototype,
+                                          bool set_missing_required) {
+  // Null outer context means it's either already verified or unverified.
+  //
+  // If the payload is eagerly verified, the recursion limit was also verified
+  // and we don't need to repeat that. Also, users might have used a custom
+  // limit which is not known at this access.
+  //
+  // Unverified lazy fields may suffer from stack overflow with deeply nested
+  // data. We argue that it should be better than silent data corruption.
+  constexpr int kUnlimitedDepth = std::numeric_limits<int>::max();
+  const char* ptr;
+  ParseContext local_ctx(kUnlimitedDepth, false, &ptr, input);
+
+  if (set_missing_required) {
+    local_ctx.SetParentMissingRequiredFields();
+  }
+  // Unparsed data is already verified at parsing. Disable eager-verification.
+  (void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy);
+
+  ptr = InternalParseRepeated(ptr, &local_ctx, value, prototype);
+  return ptr != nullptr &&
+         (local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit());
+}
+
+template <typename T>
+inline bool ParseWithOuterContextImpl(const T& input, ParseContext* ctx,
+                                      RepeatedPtrFieldBase* value,
+                                      const MessageLite* prototype,
+                                      bool set_missing_required) {
+  if (ctx == nullptr) {
+    return ParseWithNullOuterContextImpl(input, value, prototype,
+                                         set_missing_required);
+  }
+
+  ABSL_DCHECK(!ctx->AliasingEnabled());
+  // set_missing_required => ctx == nullptr
+  ABSL_DCHECK(!set_missing_required);
+
+  // Create local context with depth.
+  const char* ptr;
+  ParseContext local_ctx(ParseContext::kSpawn, *ctx, &ptr, input);
+
+  if (set_missing_required) {
+    local_ctx.SetParentMissingRequiredFields();
+  }
+  if (ctx->lazy_parse_mode() == ParseContext::LazyParseMode::kEagerVerify) {
+    // Unparsed data is already verified at parsing. Disable eager-verification.
+    (void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy);
+  }
+
+  ptr = InternalParseRepeated(ptr, &local_ctx, value, prototype);
+
+  if (local_ctx.missing_required_fields()) {
+    ctx->SetMissingRequiredFields();
+  }
+
+  return ptr != nullptr &&
+         (local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit());
+}
+
+class ByPrototype {
+ public:
+  explicit ByPrototype(const MessageLite* prototype) : prototype_(prototype) {}
+
+  MessageLite* New(Arena* arena) const { return prototype_->New(arena); }
+
+  const MessageLite& Default() const { return *prototype_; }
+
+ private:
+  const MessageLite* prototype_;
+};
+}  // namespace
+
+const RepeatedPtrFieldBase* LazyRepeatedPtrField::GetByPrototype(
+    const MessageLite* prototype, Arena* arena, ParseContext* ctx) const {
+  return GetGeneric(ByPrototype(prototype), arena, ctx);
+}
+
+RepeatedPtrFieldBase* LazyRepeatedPtrField::MutableByPrototype(
+    const MessageLite* prototype, Arena* arena, ParseContext* ctx) {
+  return MutableGeneric(ByPrototype(prototype), arena, ctx);
+}
+
+void LazyRepeatedPtrField::Clear() {
+  PerformTransition([](ExclusiveTxn& txn) {
+    auto* value = txn.mutable_value();
+    if (value != nullptr) value->Clear<GenericTypeHandler<MessageLite>>();
+    return RawState::kCleared;
+  });
+}
+
+bool LazyRepeatedPtrField::IsEagerSerializeSafe(const MessageLite* prototype,
+                                                int32_t number,
+                                                Arena* arena) const {
+  // "prototype" may be null if it is for dynamic messages. This is ok as
+  // dynamic extensions won't be lazy as they lack verify functions any way.
+  if (prototype == nullptr) return false;
+
+  for (;;) {
+    switch (GetLogicalState()) {
+      case LogicalState::kClear:
+      case LogicalState::kClearExposed:
+      case LogicalState::kDirty:
+        return true;
+      case LogicalState::kNoParseRequired: {
+        const auto* value = raw_.load(std::memory_order_relaxed).value();
+        size_t tag_size = WireFormatLite::TagSize(
+            number, WireFormatLite::FieldType::TYPE_MESSAGE);
+        size_t total_size = tag_size * value->size();
+        for (int i = 0; i < value->size(); i++) {
+          total_size += WireFormatLite::LengthDelimitedSize(
+              value->Get<GenericTypeHandler<MessageLite>>(i).ByteSizeLong());
+        }
+        return total_size == unparsed_.Size();
+      }
+      case LogicalState::kParseRequired: {
+        GetByPrototype(prototype, arena);
+        break;  // reswitch
+      }
+    }
+  }
+  // Required for certain compiler configurations.
+  ABSL_LOG(FATAL) << "Not reachable";
+  return false;
+}
+
+void LazyRepeatedPtrField::swap_atomics(std::atomic<MessageState>& lhs,
+                                        std::atomic<MessageState>& rhs) {
+  auto l = lhs.exchange(rhs.load(std::memory_order_relaxed),
+                        std::memory_order_relaxed);
+  rhs.store(l, std::memory_order_relaxed);
+}
+
+void LazyRepeatedPtrField::Swap(LazyRepeatedPtrField* lhs, Arena* lhs_arena,
+                                LazyRepeatedPtrField* rhs, Arena* rhs_arena) {
+  static auto reallocate = [](LazyRepeatedPtrField* f, Arena* arena,
+                              bool cleanup_old) {
+    auto raw = f->raw_.load(std::memory_order_relaxed);
+    if (raw.value() != nullptr) {
+      auto* new_value = Arena::Create<RepeatedPtrFieldBase>(arena);
+      if (!raw.value()->empty()) {
+        new_value->MergeFrom<MessageLite>(*raw.value());
+      }
+      if (cleanup_old) {
+        delete reinterpret_cast<const RepeatedPtrField<MessageLite>*>(
+            raw.value());
+      };
+      raw.set_value(new_value);
+      f->raw_.store(raw, std::memory_order_relaxed);
+    }
+    auto old_unparsed = f->unparsed_;
+    f->unparsed_.Visit(
+        [] {},
+        [&](auto& cord) { f->unparsed_.InitAsCord(arena, std::move(cord)); },
+        [&](auto view) {
+          if (arena == nullptr) {
+            f->unparsed_.InitAsCord(arena, view);
+          } else {
+            f->unparsed_.InitAndSetArray(arena, view);
+          }
+        });
+    if (cleanup_old) old_unparsed.Destroy();
+  };
+  static auto take_ownership = [](LazyRepeatedPtrField* f, Arena* arena) {
+#ifdef PROTOBUF_FORCE_COPY_IN_SWAP
+    reallocate(f, arena, true);
+#else
+    arena->Own(reinterpret_cast<RepeatedPtrField<MessageLite>*>(
+        f->raw_.load(std::memory_order_relaxed).mutable_value()));
+    f->unparsed_.TransferHeapOwnershipToArena(arena);
+#endif
+  };
+
+  using std::swap;  // Enable ADL with fallback
+  swap_atomics(lhs->raw_, rhs->raw_);
+  swap(lhs->unparsed_, rhs->unparsed_);
+  // At this point we are in a weird state.  The messages have been swapped into
+  // their destination, but we have completely ignored the arenas, so the owning
+  // arena is actually on the opposite message.  Now we straighten out our
+  // ownership by forcing reallocations/ownership changes as needed.
+  if (lhs_arena == rhs_arena) {
+#ifdef PROTOBUF_FORCE_COPY_IN_SWAP
+    if (lhs_arena == nullptr) {
+      reallocate(lhs, lhs_arena, true);
+      reallocate(rhs, rhs_arena, true);
+    }
+#endif
+  } else {
+    if (lhs_arena == nullptr) {
+      take_ownership(rhs, rhs_arena);
+      reallocate(lhs, lhs_arena, false);
+    } else if (rhs_arena == nullptr) {
+      take_ownership(lhs, lhs_arena);
+      reallocate(rhs, rhs_arena, false);
+    } else {
+      reallocate(lhs, lhs_arena, false);
+      reallocate(rhs, rhs_arena, false);
+    }
+  }
+}
+
+void LazyRepeatedPtrField::InternalSwap(
+    LazyRepeatedPtrField* PROTOBUF_RESTRICT lhs,
+    LazyRepeatedPtrField* PROTOBUF_RESTRICT rhs) {
+  using std::swap;  // Enable ADL with fallback
+  swap_atomics(lhs->raw_, rhs->raw_);
+  swap(lhs->unparsed_, rhs->unparsed_);
+}
+
+bool LazyRepeatedPtrField::ParseWithOuterContext(RepeatedPtrFieldBase* value,
+                                                 const absl::Cord& input,
+                                                 ParseContext* ctx,
+                                                 const MessageLite* prototype,
+                                                 bool set_missing_required) {
+  absl::optional<absl::string_view> flat = input.TryFlat();
+  if (flat.has_value()) {
+    return ParseWithOuterContextImpl(*flat, ctx, value, prototype,
+                                     set_missing_required);
+  }
+
+  io::CordInputStream cis(&input);
+  return ParseWithOuterContextImpl(&cis, ctx, value, prototype,
+                                   set_missing_required);
+}
+
+bool LazyRepeatedPtrField::ParseWithOuterContext(RepeatedPtrFieldBase* value,
+                                                 absl::string_view input,
+                                                 ParseContext* ctx,
+                                                 const MessageLite* prototype,
+                                                 bool set_missing_required) {
+  return ParseWithOuterContextImpl(input, ctx, value, prototype,
+                                   set_missing_required);
+}
+
+size_t LazyRepeatedPtrField::ByteSizeLong(size_t tag_size) const {
+  switch (GetLogicalState()) {
+    case LogicalState::kClear:
+    case LogicalState::kClearExposed:
+    case LogicalState::kNoParseRequired:
+    case LogicalState::kParseRequired:
+      return unparsed_.Size();
+
+    case LogicalState::kDirty:
+      const auto* value = raw_.load(std::memory_order_relaxed).value();
+      size_t total_size = tag_size * value->size();
+      for (int i = 0; i < value->size(); i++) {
+        total_size += WireFormatLite::LengthDelimitedSize(
+            value->Get<GenericTypeHandler<MessageLite>>(i).ByteSizeLong());
+      }
+      return total_size;
+  }
+  // Required for certain compiler configurations.
+  ABSL_LOG(FATAL) << "Not reachable";
+  return -1;
+}
+
+void LazyRepeatedPtrField::LogParseError(const RepeatedPtrFieldBase* value) {
+  const MessageLite* message =
+      &value->at<internal::GenericTypeHandler<MessageLite>>(0);
+  auto get_error_string = [&value]() {
+    std::string str;
+    for (int i = 0; i < value->size(); i++) {
+      absl::StrAppend(&str, "[", i, "]: ",
+                      value->at<internal::GenericTypeHandler<MessageLite>>(i)
+                          .InitializationErrorString(),
+                      "\n");
+    }
+    return str;
+  };
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
+  // In fuzzing mode, we log less to speed up fuzzing.
+  ABSL_LOG_EVERY_N(INFO, 100000)
+#else
+  ABSL_LOG_EVERY_N_SEC(INFO, 1)
+#endif
+      << "Lazy parsing failed for RepeatedPtrField<" << message->GetTypeName()
+      << "> error=" << get_error_string() << " (N = " << COUNTER << ")";
+}
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
+
+#include "google/protobuf/port_undef.inc"

src/google/protobuf/lazy_repeated_field_heavy.cc

@@ -0,0 +1,401 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2023 Google Inc.  All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
+#include "absl/strings/cord.h"
+#include "absl/strings/escaping.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_replace.h"
+#include "absl/strings/string_view.h"
+#include "google/protobuf/arena.h"
+#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
+#include "google/protobuf/lazy_repeated_field.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/message_lite.h"
+#include "google/protobuf/repeated_ptr_field.h"
+#include "google/protobuf/wire_format_lite.h"
+
+namespace google {
+namespace protobuf {
+namespace internal {
+namespace {
+
+class ByFactory {
+ public:
+  explicit ByFactory(const Descriptor* type, MessageFactory* factory)
+      : type_(type), factory_(factory) {}
+
+  Message* New(Arena* arena) const {
+    return factory_->GetPrototype(type_)->New(arena);
+  }
+
+  const Message& Default() const { return *factory_->GetPrototype(type_); }
+
+ private:
+  const Descriptor* type_;
+  MessageFactory* factory_;
+};
+
+// Escape C++ trigraphs by escaping question marks to \?
+std::string EscapeTrigraphs(absl::string_view to_escape) {
+  return absl::StrReplaceAll(to_escape, {{"?", "\\?"}});
+}
+
+std::string EscapeEncoded(absl::string_view encoded) {
+  std::string out;
+  out.reserve(encoded.size() * 2);
+  constexpr size_t kBytesPerLine = 25;
+  for (size_t i = 0; i < encoded.size(); i += kBytesPerLine) {
+    absl::StrAppend(
+        &out, "\"",
+        EscapeTrigraphs(absl::CEscape(encoded.substr(i, kBytesPerLine))),
+        "\"\n");
+  }
+  return out;
+}
+
+// Deterministic serialization is required to minimize false positives: e.g.
+// ordering, redundant wire format data, etc. Such discrepancies are
+// expected and tolerated. To prevent this serialization starts yet another
+// consistency check, we should skip consistency-check.
+std::string DeterministicSerialization(const google::protobuf::MessageLite& m) {
+  std::string result;
+  {
+    google::protobuf::io::StringOutputStream sink(&result);
+    google::protobuf::io::CodedOutputStream out(&sink);
+    out.SetSerializationDeterministic(true);
+    out.SkipCheckConsistency();
+    m.SerializePartialToCodedStream(&out);
+  }
+  return result;
+}
+
+// If LazyField is initialized, unparsed and message should be consistent. If
+// a LazyField is mutated via const_cast, that may break. We should rather fail
+// than silently propagate such discrepancy. Note that this aims to detect
+// missing/added data.
+void VerifyConsistency(LazyRepeatedPtrField::LogicalState state,
+                       const RepeatedPtrFieldBase* value,
+                       const MessageLite* prototype, const absl::Cord& unparsed,
+                       io::EpsCopyOutputStream* stream) {
+#ifndef NDEBUG
+  if (stream != nullptr && !stream->ShouldCheckConsistency()) return;
+  if (state != LazyRepeatedPtrField::LogicalState::kNoParseRequired) return;
+
+  RepeatedPtrField<Message> unparsed_msg;
+  if (!LazyRepeatedPtrField::ParseWithOuterContext(
+          reinterpret_cast<RepeatedPtrFieldBase*>(&unparsed_msg), unparsed,
+          nullptr, prototype, /*set_missing_required=*/false)) {
+    // Bail out on parse failure as it can result in false positive
+    // inconsistency and ABSL_CHECK failure. Warn instead.
+    ABSL_LOG(WARNING)
+        << "Verify skipped due to parse falure: RepeatedPtrField of "
+        << prototype->GetTypeName();
+    return;
+  }
+
+  const auto* msgs = reinterpret_cast<const RepeatedPtrField<Message>*>(value);
+  // Eagerly parse all lazy fields to eliminate non-canonical wireformat data.
+  for (int i = 0; i < msgs->size(); i++) {
+    // Clone a new one from the original to eagerly parse all lazy
+    // fields.
+    const auto& msg = msgs->Get(i);
+    std::unique_ptr<Message> clone(msg.New());
+    clone->CopyFrom(msg);
+    EagerParseLazyFieldIgnoreUnparsed(*clone);
+    EagerParseLazyFieldIgnoreUnparsed(*unparsed_msg.Mutable(i));
+    ABSL_DCHECK_EQ(DeterministicSerialization(*clone),
+                   DeterministicSerialization(unparsed_msg.Get(i)))
+        << "RepeatedPtrField<" << msg.GetTypeName() << ">(" << i << ")"
+        << ": likely mutated via getters + const_cast\n"
+        << "unparsed:\n"
+        << EscapeEncoded(DeterministicSerialization(unparsed_msg.Get(i)))
+        << "\nmessage:\n"
+        << EscapeEncoded(DeterministicSerialization(*clone));
+  }
+#endif  // !NDEBUG
+}
+
+}  // namespace
+
+LazyRepeatedPtrField::LazyRepeatedPtrField(Arena* arena,
+                                           const LazyRepeatedPtrField& rhs,
+                                           Arena* rhs_arena)
+    : raw_(MessageState(RawState::kCleared)) {
+  switch (rhs.GetLogicalState()) {
+    case LogicalState::kClear:
+    case LogicalState::kClearExposed:
+      return;  // Leave uninitialized / empty
+    case LogicalState::kNoParseRequired:
+    case LogicalState::kParseRequired: {
+      rhs.unparsed_.Visit(
+          [] {},  //
+          [&](const auto& cord) { unparsed_.InitAsCord(arena, cord); },
+          [&](auto view) {
+            if (arena == nullptr) {
+              unparsed_.InitAsCord(nullptr, view);
+            } else {
+              unparsed_.InitAndSetArray(arena, view);
+            }
+          });
+      raw_.store(
+          MessageState(nullptr, rhs.MaybeUninitialized()
+                                    ? RawState::kNeedsParseMaybeUninitialized
+                                    : RawState::kNeedsParse),
+          std::memory_order_relaxed);
+      return;
+    }
+    case LogicalState::kDirty: {
+      MessageState state = rhs.raw_.load(std::memory_order_relaxed);
+      const auto* src = state.value();
+      if (src->empty()) {
+        return;  // Leave uninitialized / empty
+      }
+      // Retain the existing IsParsed or IsParsedMaybeUninitialized status.
+      // TODO: use copy construction.
+      auto new_state = state.status();
+      auto* value = Arena::Create<RepeatedPtrFieldBase>(arena);
+      // MergeFrom calls reserve.
+      value->MergeFrom<MessageLite>(*src);
+      raw_.store(MessageState(value, new_state), std::memory_order_relaxed);
+      return;
+    }
+  }
+}
+
+const RepeatedPtrFieldBase* LazyRepeatedPtrField::GetDynamic(
+    const Descriptor* type, MessageFactory* factory, Arena* arena) const {
+  return GetGeneric(ByFactory(type, factory), arena, nullptr);
+}
+
+RepeatedPtrFieldBase* LazyRepeatedPtrField::MutableDynamic(
+    const Descriptor* type, MessageFactory* factory, Arena* arena) {
+  return MutableGeneric(ByFactory(type, factory), arena, nullptr);
+}
+
+size_t LazyRepeatedPtrField::SpaceUsedExcludingSelfLong() const {
+  // absl::Cord::EstimatedMemoryUsage counts itself that should be excluded
+  // because sizeof(Cord) is already counted in self.
+  size_t total_size = unparsed_.SpaceUsedExcludingSelf();
+  switch (GetLogicalState()) {
+    case LogicalState::kClearExposed:
+    case LogicalState::kNoParseRequired:
+    case LogicalState::kDirty: {
+      const auto* value = raw_.load(std::memory_order_relaxed).value();
+      total_size +=
+          value->SpaceUsedExcludingSelfLong<GenericTypeHandler<Message>>();
+    } break;
+    case LogicalState::kClear:
+    case LogicalState::kParseRequired:
+      // We may have a `Message*` here, but we cannot safely access it
+      // because, a racing SharedInit could delete it out from under us.
+      // Other states in this structure are already passed kSharedInit and are
+      // thus safe.
+      break;  // Nothing to add.
+  }
+  return total_size;
+}
+
+template <typename Input>
+bool LazyRepeatedPtrField::MergeFrom(const MessageLite* prototype,
+                                     const Input& data, Arena* arena) {
+  switch (GetLogicalState()) {
+    case LogicalState::kParseRequired: {
+      unparsed_.UpgradeToCord(arena).Append(data);
+      break;
+    }
+    case LogicalState::kClear: {
+      size_t num_bytes = data.size();
+      ABSL_DCHECK(num_bytes > 0);
+      if (arena == nullptr || num_bytes > kMaxArraySize || unparsed_.IsCord()) {
+        unparsed_.SetCord(arena, data);
+      } else {
+        unparsed_.InitAndSetArray(arena, data);
+      }
+      SetNeedsParse();
+      break;
+    }
+
+    // Pointer was previously exposed merge into that object.
+    case LogicalState::kClearExposed:
+    case LogicalState::kNoParseRequired:
+    case LogicalState::kDirty: {
+      auto new_state = PerformTransition([&](ExclusiveTxn& txn) {
+        auto* value = txn.mutable_value();
+        bool res =
+            ParseWithOuterContext(value, data, /*ctx=*/nullptr, prototype,
+                                  /*set_missing_required=*/false);
+        if (!res) {
+          LogParseError(value);
+          return RawState::kParseError;
+        } else {
+          return RawState::kIsParsed;
+        }
+      });
+      return new_state == RawState::kIsParsed;
+    }
+  }
+  return true;
+}
+
+void LazyRepeatedPtrField::MergeMaybeUninitializedState(
+    const LazyRepeatedPtrField& other) {
+  if (MaybeUninitialized() || !other.MaybeUninitialized()) return;
+
+  switch (GetLogicalState()) {
+    case LogicalState::kParseRequired:
+      SetNeedsParseMaybeUninitialized();
+      break;
+    case LogicalState::kNoParseRequired:
+      SetParseNotRequiredMaybeUninitialized();
+      break;
+    default:
+      break;
+  }
+}
+
+void LazyRepeatedPtrField::MergeFrom(const MessageLite* prototype,
+                                     const LazyRepeatedPtrField& other,
+                                     Arena* arena, Arena* other_arena) {
+#ifndef NDEBUG
+  VerifyConsistency(other.GetLogicalState(),
+                    other.raw_.load(std::memory_order_relaxed).value(),
+                    prototype, other.unparsed_.ForceAsCord(), nullptr);
+#endif  // !NDEBUG
+  switch (other.GetLogicalState()) {
+    case LogicalState::kClear:
+    case LogicalState::kClearExposed:
+      return;  // Nothing to do.
+
+    case LogicalState::kParseRequired:
+    case LogicalState::kNoParseRequired:
+      if (other.unparsed_.IsCord()) {
+        MergeFrom(prototype, other.unparsed_.AsCord(), arena);
+      } else {
+        MergeFrom(prototype, other.unparsed_.AsStringView(), arena);
+      }
+      MergeMaybeUninitializedState(other);
+      return;
+
+    case LogicalState::kDirty: {
+      const auto* other_value =
+          other.raw_.load(std::memory_order_relaxed).value();
+      if (other_value->empty()) {
+        return;
+      }
+      auto* value = MutableByPrototype(prototype, arena);
+      value->MergeFrom<MessageLite>(*other_value);
+      // No need to merge uninitialized state.
+      ABSL_DCHECK(GetLogicalState() == LogicalState::kDirty);
+      return;
+    }
+  }
+}
+
+uint8_t* LazyRepeatedPtrField::InternalWrite(
+    const MessageLite* prototype, int32_t number, uint8_t* target,
+    io::EpsCopyOutputStream* stream) const {
+#ifndef NDEBUG
+  VerifyConsistency(GetLogicalState(),
+                    raw_.load(std::memory_order_relaxed).value(), prototype,
+                    unparsed_.ForceAsCord(), stream);
+#endif  // !NDEBUG
+  switch (GetLogicalState()) {
+    case LogicalState::kClear:
+    case LogicalState::kClearExposed:
+    case LogicalState::kNoParseRequired:
+    case LogicalState::kParseRequired:
+      // If deterministic is enabled then attempt to parse to a message which
+      // can then be serialized deterministically. (The serialized bytes may
+      // have been created undeterministically).
+      if (stream->IsSerializationDeterministic() && prototype != nullptr) {
+        RepeatedPtrField<MessageLite> value;
+        // TODO: Test this path.
+        bool success = unparsed_.Visit(
+            [] { return true; },
+            [&](const auto& cord) {
+              // `set_missing_required = false` to avoid checking require fields
+              // (simialr to Message::ParsePartial*).
+              return ParseWithOuterContext(
+                  reinterpret_cast<RepeatedPtrFieldBase*>(&value), cord,
+                  /*ctx=*/nullptr, prototype, /*set_missing_required=*/false);
+            },
+            [&](auto view) {
+              return ParseWithOuterContext(
+                  reinterpret_cast<RepeatedPtrFieldBase*>(&value), view,
+                  /*ctx=*/nullptr, prototype, false);
+            });
+        if (success) {
+          size_t tag_size = WireFormatLite::TagSize(
+              number, WireFormatLite::FieldType::TYPE_MESSAGE);
+          auto count = tag_size * value.size();
+          for (int i = 0; i < value.size(); i++) {
+            count += WireFormatLite::LengthDelimitedSize(
+                value.Get(i).ByteSizeLong());
+          }
+
+          // Serialization takes place in two phases:
+          // 1) Figure out the expected number of bytes (e.g. ByteSizeLong() on
+          // the container message) 2) InternalWrite the bytes.
+          //
+          // There is a golden contract that the # of bytes written matches
+          // the returned value from the first step.
+          //
+          // In this case unparsed_ was used as the source of truth for the
+          // number of bytes. There are some known cases where the number of
+          // serialized bytes is different than the number of bytes written
+          // by a message parsed from the serialized bytes. For example if the
+          // serialized representation contained multiple entries for the same
+          // non-repeated field the duplicates are removed upon parsing.
+          //
+          // If this (relatively rare) case is hit then there is no choice
+          // but to serialize the original unparsed bytes; otherwise the
+          // golden contract is broken.
+          // It's possible for the size to change if the unparsed_ was not
+          // canonical, for example it can have repeated entries for the same
+          // tag (this is more common then you would think).
+          if (count == unparsed_.Size()) {
+            for (int i = 0, n = value.size(); i < n; i++) {
+              const auto& repfield = value.Get(i);
+              target = WireFormatLite::InternalWriteMessage(
+                  number, repfield, repfield.GetCachedSize(), target, stream);
+            }
+            return target;
+          }
+        }
+      }
+      return unparsed_.Visit(
+          [&] { return target; },
+          [&](const auto& cord) { return stream->WriteCord(cord, target); },
+          [&](auto view) {
+            return stream->WriteRaw(view.data(), view.size(), target);
+          });
+    case LogicalState::kDirty: {
+      const auto* value = raw_.load(std::memory_order_relaxed).value();
+      for (int i = 0, n = value->size(); i < n; i++) {
+        const auto& repfield = value->Get<GenericTypeHandler<MessageLite>>(i);
+        target = WireFormatLite::InternalWriteMessage(
+            number, repfield, repfield.GetCachedSize(), target, stream);
+      }
+      return target;
+    }
+  }
+  // Required for certain compiler configurations.
+  ABSL_LOG(FATAL) << "Not reachable";
+  return nullptr;
+}
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/repeated_ptr_field.h

@@ -576,6 +576,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
   // subclass.
   friend class google::protobuf::Reflection;
   friend class internal::SwapFieldHelper;
+  friend class LazyRepeatedPtrField;
 
   // Concrete Arena enabled copy function used to copy messages instances.
   // This follows the `Arena::CopyConstruct` signature so that the compiler