From 7c2ed10beb0dccbbfb6a76e43d28647794d8d7e7 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 2 Aug 2023 08:45:34 -0700 Subject: [PATCH] Overload the void* in RepeatedPtrFieldBase to point directly to the first element if Capacity()==1, or to point to an allocated Rep if Capacity()>1. This avoids allocating memory for 1 element containers, and avoids indirection costs like extra cache misses. PiperOrigin-RevId: 553157394 --- src/google/protobuf/implicit_weak_message.h | 4 +- .../protobuf/repeated_field_unittest.cc | 69 +++-- src/google/protobuf/repeated_ptr_field.cc | 132 ++++++--- src/google/protobuf/repeated_ptr_field.h | 251 +++++++++++------- 4 files changed, 297 insertions(+), 159 deletions(-) diff --git a/src/google/protobuf/implicit_weak_message.h b/src/google/protobuf/implicit_weak_message.h index 71d05b91fa..b9ff1fa28d 100644 --- a/src/google/protobuf/implicit_weak_message.h +++ b/src/google/protobuf/implicit_weak_message.h @@ -198,13 +198,13 @@ struct WeakRepeatedPtrField { return pointer_iterator(base().raw_mutable_data()); } const_pointer_iterator pointer_begin() const { - return const_pointer_iterator(base().raw_mutable_data()); + return const_pointer_iterator(base().raw_data()); } pointer_iterator pointer_end() { return pointer_iterator(base().raw_mutable_data() + base().size()); } const_pointer_iterator pointer_end() const { - return const_pointer_iterator(base().raw_mutable_data() + base().size()); + return const_pointer_iterator(base().raw_data() + base().size()); } MessageLite* AddWeak(const MessageLite* prototype) { diff --git a/src/google/protobuf/repeated_field_unittest.cc b/src/google/protobuf/repeated_field_unittest.cc index fceb642ba5..5ccfa34ba3 100644 --- a/src/google/protobuf/repeated_field_unittest.cc +++ b/src/google/protobuf/repeated_field_unittest.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -886,13 +887,9 @@ TEST(RepeatedField, MoveAssign) { Arena::CreateMessage>(&arena); destination->Add(3); const int* source_data = source->data(); - const int* destination_data = destination->data(); *destination = std::move(*source); EXPECT_EQ(source_data, destination->data()); EXPECT_THAT(*destination, ElementsAre(1, 2)); - // This property isn't guaranteed but it's useful to have a test that would - // catch changes in this area. - EXPECT_EQ(destination_data, source->data()); EXPECT_THAT(*source, ElementsAre(3)); } { @@ -1656,9 +1653,18 @@ TEST(RepeatedPtrField, AddAllocated) { field.Add()->assign("filler"); } + const auto ensure_at_capacity = [&] { + while (field.size() < field.Capacity()) { + field.Add()->assign("filler"); + } + }; + const auto ensure_not_at_capacity = [&] { field.Reserve(field.size() + 1); }; + + ensure_at_capacity(); int index = field.size(); // First branch: Field is at capacity with no cleared objects. + ASSERT_EQ(field.size(), field.Capacity()); std::string* foo = new std::string("foo"); field.AddAllocated(foo); EXPECT_EQ(index + 1, field.size()); @@ -1666,6 +1672,7 @@ TEST(RepeatedPtrField, AddAllocated) { EXPECT_EQ(foo, &field.Get(index)); // Last branch: Field is not at capacity and there are no cleared objects. + ensure_not_at_capacity(); std::string* bar = new std::string("bar"); field.AddAllocated(bar); ++index; @@ -1674,6 +1681,7 @@ TEST(RepeatedPtrField, AddAllocated) { EXPECT_EQ(bar, &field.Get(index)); // Third branch: Field is not at capacity and there are no cleared objects. + ensure_not_at_capacity(); field.RemoveLast(); std::string* baz = new std::string("baz"); field.AddAllocated(baz); @@ -1682,9 +1690,7 @@ TEST(RepeatedPtrField, AddAllocated) { EXPECT_EQ(baz, &field.Get(index)); // Second branch: Field is at capacity but has some cleared objects. - while (field.size() < field.Capacity()) { - field.Add()->assign("filler2"); - } + ensure_at_capacity(); field.RemoveLast(); index = field.size(); std::string* moo = new std::string("moo"); @@ -1842,6 +1848,47 @@ TEST(RepeatedPtrField, IteratorConstruct_Proto) { EXPECT_EQ(values[1].bb(), other.Get(1).bb()); } +TEST(RepeatedPtrField, SmallOptimization) { + // Properties checked here are not part of the contract of RepeatedPtrField, + // but we test them to verify that SSO is working as expected by the + // implementation. + + // We use an arena to easily measure memory usage, but not needed. + Arena arena; + auto* array = Arena::CreateMessage>(&arena); + EXPECT_EQ(array->Capacity(), 1); + EXPECT_EQ(array->SpaceUsedExcludingSelf(), 0); + std::string str; + auto usage_before = arena.SpaceUsed(); + // We use UnsafeArenaAddAllocated just to grow the array without creating + // objects or causing extra cleanup costs in the arena to make the + // measurements simpler. + array->UnsafeArenaAddAllocated(&str); + // No backing array, just the string. + EXPECT_EQ(array->SpaceUsedExcludingSelf(), sizeof(str)); + // We have not used any arena space. + EXPECT_EQ(usage_before, arena.SpaceUsed()); + // Verify the string is where we think it is. + EXPECT_EQ(&*array->begin(), &str); + EXPECT_EQ(array->pointer_begin()[0], &str); + // The T** in pointer_begin points into the sso in the object. + EXPECT_TRUE(std::less_equal{}(array, &*array->pointer_begin())); + EXPECT_TRUE(std::less_equal{}(&*array->pointer_begin(), array + 1)); + + // Adding a second object stops sso. + std::string str2; + array->UnsafeArenaAddAllocated(&str2); + EXPECT_EQ(array->Capacity(), 3); + // Backing array and the strings. + EXPECT_EQ(array->SpaceUsedExcludingSelf(), + (1 + array->Capacity()) * sizeof(void*) + 2 * sizeof(str)); + // We used some arena space now. + EXPECT_LT(usage_before, arena.SpaceUsed()); + // And the pointer_begin is not in the sso anymore. + EXPECT_FALSE(std::less_equal{}(array, &*array->pointer_begin()) && + std::less_equal{}(&*array->pointer_begin(), array + 1)); +} + TEST(RepeatedPtrField, CopyAssign) { RepeatedPtrField source, destination; source.Add()->assign("4"); @@ -1907,13 +1954,9 @@ TEST(RepeatedPtrField, MoveAssign) { RepeatedPtrField destination; *destination.Add() = "3"; const std::string* const* source_data = source.data(); - const std::string* const* destination_data = destination.data(); destination = std::move(source); EXPECT_EQ(source_data, destination.data()); EXPECT_THAT(destination, ElementsAre("1", "2")); - // This property isn't guaranteed but it's useful to have a test that would - // catch changes in this area. - EXPECT_EQ(destination_data, source.data()); EXPECT_THAT(source, ElementsAre("3")); } { @@ -1926,13 +1969,9 @@ TEST(RepeatedPtrField, MoveAssign) { Arena::CreateMessage>(&arena); *destination->Add() = "3"; const std::string* const* source_data = source->data(); - const std::string* const* destination_data = destination->data(); *destination = std::move(*source); EXPECT_EQ(source_data, destination->data()); EXPECT_THAT(*destination, ElementsAre("1", "2")); - // This property isn't guaranteed but it's useful to have a test that would - // catch changes in this area. - EXPECT_EQ(destination_data, source->data()); EXPECT_THAT(*source, ElementsAre("3")); } { diff --git a/src/google/protobuf/repeated_ptr_field.cc b/src/google/protobuf/repeated_ptr_field.cc index 42a77d7549..463561fc1d 100644 --- a/src/google/protobuf/repeated_ptr_field.cc +++ b/src/google/protobuf/repeated_ptr_field.cc @@ -33,8 +33,12 @@ // Sanjay Ghemawat, Jeff Dean, and others. #include +#include +#include +#include #include "absl/log/absl_check.h" +#include "google/protobuf/arena.h" #include "google/protobuf/implicit_weak_message.h" #include "google/protobuf/port.h" #include "google/protobuf/repeated_field.h" @@ -52,45 +56,58 @@ void** RepeatedPtrFieldBase::InternalExtend(int extend_amount) { if (total_size_ >= new_size) { // N.B.: rep_ is non-nullptr because extend_amount is always > 0, hence // total_size must be non-zero since it is lower-bounded by new_size. - return &rep_->elements[current_size_]; + return elements() + current_size_; } - Rep* old_rep = rep_; + Arena* arena = GetOwningArena(); new_size = internal::CalculateReserveSize(total_size_, new_size); ABSL_CHECK_LE(static_cast(new_size), static_cast( (std::numeric_limits::max() - kRepHeaderSize) / - sizeof(old_rep->elements[0]))) + sizeof(rep()->elements[0]))) << "Requested size is too large to fit into size_t."; - size_t bytes = kRepHeaderSize + sizeof(old_rep->elements[0]) * new_size; + size_t bytes = kRepHeaderSize + sizeof(rep()->elements[0]) * new_size; + Rep* new_rep; + void* old_tagged_ptr = tagged_rep_or_elem_; if (arena == nullptr) { internal::SizedPtr res = internal::AllocateAtLeast(bytes); - new_size = (res.n - kRepHeaderSize) / sizeof(old_rep->elements[0]); - rep_ = reinterpret_cast(res.p); + new_size = + static_cast((res.n - kRepHeaderSize) / sizeof(rep()->elements[0])); + new_rep = reinterpret_cast(res.p); } else { - rep_ = reinterpret_cast(Arena::CreateArray(arena, bytes)); + new_rep = reinterpret_cast(Arena::CreateArray(arena, bytes)); } - const int old_total_size = total_size_; - total_size_ = new_size; - if (old_rep) { - if (old_rep->allocated_size > 0) { - memcpy(rep_->elements, old_rep->elements, - old_rep->allocated_size * sizeof(rep_->elements[0])); - } - rep_->allocated_size = old_rep->allocated_size; - const size_t old_size = - old_total_size * sizeof(rep_->elements[0]) + kRepHeaderSize; - if (arena == nullptr) { - internal::SizedDelete(old_rep, old_size); + if (using_sso()) { + new_rep->elements[0] = old_tagged_ptr; + new_rep->allocated_size = old_tagged_ptr != nullptr ? 1 : 0; + } else { + if (old_tagged_ptr) { + Rep* old_rep = reinterpret_cast( + reinterpret_cast(old_tagged_ptr) - 1); + if (old_rep->allocated_size > 0) { + memcpy(new_rep->elements, old_rep->elements, + old_rep->allocated_size * sizeof(rep()->elements[0])); + } + new_rep->allocated_size = old_rep->allocated_size; + + const size_t old_size = + total_size_ * sizeof(rep()->elements[0]) + kRepHeaderSize; + if (arena == nullptr) { + internal::SizedDelete(old_rep, old_size); + } else { + arena_->ReturnArrayMemory(old_rep, old_size); + } } else { - arena_->ReturnArrayMemory(old_rep, old_size); + new_rep->allocated_size = 0; } - } else { - rep_->allocated_size = 0; } - return &rep_->elements[current_size_]; + + tagged_rep_or_elem_ = + reinterpret_cast(reinterpret_cast(new_rep) + 1); + total_size_ = new_size; + return &new_rep->elements[current_size_]; } void RepeatedPtrFieldBase::Reserve(int new_size) { @@ -100,49 +117,76 @@ void RepeatedPtrFieldBase::Reserve(int new_size) { } void RepeatedPtrFieldBase::DestroyProtos() { - ABSL_DCHECK(rep_); + ABSL_DCHECK(tagged_rep_or_elem_); ABSL_DCHECK(arena_ == nullptr); - int n = rep_->allocated_size; - void* const* elements = rep_->elements; - for (int i = 0; i < n; i++) { - delete static_cast(elements[i]); + if (using_sso()) { + delete static_cast(tagged_rep_or_elem_); + + } else { + Rep* r = rep(); + int n = r->allocated_size; + void* const* elements = r->elements; + for (int i = 0; i < n; i++) { + delete static_cast(elements[i]); + } + const size_t size = total_size_ * sizeof(elements[0]) + kRepHeaderSize; + internal::SizedDelete(r, size); + tagged_rep_or_elem_ = nullptr; } - const size_t size = total_size_ * sizeof(elements[0]) + kRepHeaderSize; - internal::SizedDelete(rep_, size); - rep_ = nullptr; } void* RepeatedPtrFieldBase::AddOutOfLineHelper(void* obj) { - if (!rep_ || rep_->allocated_size == total_size_) { + if (tagged_rep_or_elem_ == nullptr) { + ABSL_DCHECK_EQ(current_size_, 0); + ABSL_DCHECK(using_sso()); + ABSL_DCHECK_EQ(allocated_size(), 0); + ExchangeCurrentSize(1); + tagged_rep_or_elem_ = obj; + return obj; + } + if (using_sso() || rep()->allocated_size == total_size_) { InternalExtend(1); // Equivalent to "Reserve(total_size_ + 1)" } - ++rep_->allocated_size; - rep_->elements[ExchangeCurrentSize(current_size_ + 1)] = obj; + Rep* r = rep(); + ++r->allocated_size; + r->elements[ExchangeCurrentSize(current_size_ + 1)] = obj; return obj; } void RepeatedPtrFieldBase::CloseGap(int start, int num) { - if (rep_ == nullptr) return; - // Close up a gap of "num" elements starting at offset "start". - for (int i = start + num; i < rep_->allocated_size; ++i) - rep_->elements[i - num] = rep_->elements[i]; + if (using_sso()) { + if (start == 0 && num == 1) { + tagged_rep_or_elem_ = nullptr; + } + } else { + // Close up a gap of "num" elements starting at offset "start". + Rep* r = rep(); + for (int i = start + num; i < r->allocated_size; ++i) + r->elements[i - num] = r->elements[i]; + r->allocated_size -= num; + } ExchangeCurrentSize(current_size_ - num); - rep_->allocated_size -= num; } MessageLite* RepeatedPtrFieldBase::AddWeak(const MessageLite* prototype) { - if (rep_ != nullptr && current_size_ < rep_->allocated_size) { + if (current_size_ < allocated_size()) { return reinterpret_cast( - rep_->elements[ExchangeCurrentSize(current_size_ + 1)]); + element_at(ExchangeCurrentSize(current_size_ + 1))); } - if (!rep_ || rep_->allocated_size == total_size_) { + if (allocated_size() == total_size_) { Reserve(total_size_ + 1); } - ++rep_->allocated_size; MessageLite* result = prototype ? prototype->New(arena_) : Arena::CreateMessage(arena_); - rep_->elements[ExchangeCurrentSize(current_size_ + 1)] = result; + if (using_sso()) { + ExchangeCurrentSize(current_size_ + 1); + tagged_rep_or_elem_ = result; + } else { + Rep* r = rep(); + ++r->allocated_size; + r->elements[ExchangeCurrentSize(current_size_ + 1)] = result; + } return result; } diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index 28319033b1..2a98dae3d1 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -46,9 +46,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -168,11 +170,19 @@ struct IsMovable // static int SpaceUsedLong(const Type&); // }; class PROTOBUF_EXPORT RepeatedPtrFieldBase { + static constexpr int kSSOCapacity = 1; + protected: constexpr RepeatedPtrFieldBase() - : arena_(nullptr), current_size_(0), total_size_(0), rep_(nullptr) {} + : arena_(nullptr), + current_size_(0), + total_size_(kSSOCapacity), + tagged_rep_or_elem_(nullptr) {} explicit RepeatedPtrFieldBase(Arena* arena) - : arena_(arena), current_size_(0), total_size_(0), rep_(nullptr) {} + : arena_(arena), + current_size_(0), + total_size_(kSSOCapacity), + tagged_rep_or_elem_(nullptr) {} RepeatedPtrFieldBase(const RepeatedPtrFieldBase&) = delete; RepeatedPtrFieldBase& operator=(const RepeatedPtrFieldBase&) = delete; @@ -193,29 +203,29 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { const typename TypeHandler::Type& at(int index) const { ABSL_CHECK_GE(index, 0); ABSL_CHECK_LT(index, current_size_); - return *cast(rep_->elements[index]); + return *cast(element_at(index)); } template typename TypeHandler::Type& at(int index) { ABSL_CHECK_GE(index, 0); ABSL_CHECK_LT(index, current_size_); - return *cast(rep_->elements[index]); + return *cast(element_at(index)); } template typename TypeHandler::Type* Mutable(int index) { ABSL_DCHECK_GE(index, 0); ABSL_DCHECK_LT(index, current_size_); - return cast(rep_->elements[index]); + return cast(element_at(index)); } template typename TypeHandler::Type* Add( const typename TypeHandler::Type* prototype = nullptr) { - if (rep_ != nullptr && current_size_ < rep_->allocated_size) { + if (current_size_ < allocated_size()) { return cast( - rep_->elements[ExchangeCurrentSize(current_size_ + 1)]); + element_at(ExchangeCurrentSize(current_size_ + 1))); } typename TypeHandler::Type* result = TypeHandler::NewFromPrototype(prototype, arena_); @@ -227,44 +237,50 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { typename TypeHandler, typename std::enable_if::type* = nullptr> inline void Add(typename TypeHandler::Type&& value) { - if (rep_ != nullptr && current_size_ < rep_->allocated_size) { - *cast( - rep_->elements[ExchangeCurrentSize(current_size_ + 1)]) = + if (current_size_ < allocated_size()) { + *cast(element_at(ExchangeCurrentSize(current_size_ + 1))) = std::move(value); return; } - if (!rep_ || rep_->allocated_size == total_size_) { + if (allocated_size() == total_size_) { Reserve(total_size_ + 1); } - ++rep_->allocated_size; + if (!using_sso()) ++rep()->allocated_size; typename TypeHandler::Type* result = TypeHandler::New(arena_, std::move(value)); - rep_->elements[ExchangeCurrentSize(current_size_ + 1)] = result; + element_at(ExchangeCurrentSize(current_size_ + 1)) = result; } template void Delete(int index) { ABSL_DCHECK_GE(index, 0); ABSL_DCHECK_LT(index, current_size_); - TypeHandler::Delete(cast(rep_->elements[index]), arena_); + TypeHandler::Delete(cast(element_at(index)), arena_); } // Must be called from destructor. template void Destroy() { - if (rep_ != nullptr && arena_ == nullptr) { - int n = rep_->allocated_size; - void* const* elements = rep_->elements; - for (int i = 0; i < n; i++) { - TypeHandler::Delete(cast(elements[i]), nullptr); - } - const size_t size = total_size_ * sizeof(elements[0]) + kRepHeaderSize; - internal::SizedDelete(rep_, size); + if (arena_ != nullptr) return; + + if (using_sso()) { + if (tagged_rep_or_elem_ == nullptr) return; + TypeHandler::Delete(cast(tagged_rep_or_elem_), nullptr); + return; } - rep_ = nullptr; + + Rep* r = rep(); + int n = r->allocated_size; + void* const* elems = r->elements; + for (int i = 0; i < n; i++) { + TypeHandler::Delete(cast(elems[i]), nullptr); + } + internal::SizedDelete(r, total_size_ * sizeof(elems[0]) + kRepHeaderSize); } - bool NeedsDestroy() const { return rep_ != nullptr && arena_ == nullptr; } + bool NeedsDestroy() const { + return tagged_rep_or_elem_ != nullptr && arena_ == nullptr; + } void DestroyProtos(); // implemented in the cc file public: @@ -276,7 +292,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { const typename TypeHandler::Type& Get(int index) const { ABSL_DCHECK_GE(index, 0); ABSL_DCHECK_LT(index, current_size_); - return *cast(rep_->elements[index]); + return *cast(element_at(index)); } // Creates and adds an element using the given prototype, without introducing @@ -317,31 +333,23 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { } // Prepares the container for adding elements via `AddAllocatedForParse`. - // It ensures some invariants to avoid checking then in the Add loop: - // - rep_ is not null. - // - there are no preallocated elements. + // It ensures we have no preallocated elements in the array. // Returns true if the invariants hold and `AddAllocatedForParse` can be // used. - bool PrepareForParse() { - if (current_size_ == total_size_) { - InternalExtend(1); - } - return rep_->allocated_size == current_size_; - } + bool PrepareForParse() { return allocated_size() == current_size_; } // Similar to `AddAllocated` but faster. // Can only be invoked after a call to `PrepareForParse` that returned `true`, // or other calls to `AddAllocatedForParse`. template void AddAllocatedForParse(typename TypeHandler::Type* value) { - PROTOBUF_ASSUME(rep_ != nullptr); - PROTOBUF_ASSUME(current_size_ == rep_->allocated_size); + ABSL_DCHECK_EQ(current_size_, allocated_size()); if (current_size_ == total_size_) { // The array is completely full with no cleared objects, so grow it. InternalExtend(1); } - rep_->elements[current_size_++] = value; - ++rep_->allocated_size; + element_at(current_size_++) = value; + if (!using_sso()) ++rep()->allocated_size; } protected: @@ -349,7 +357,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { void RemoveLast() { ABSL_DCHECK_GT(current_size_, 0); ExchangeCurrentSize(current_size_ - 1); - TypeHandler::Clear(cast(rep_->elements[current_size_])); + TypeHandler::Clear(cast(element_at(current_size_))); } template @@ -372,10 +380,8 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { } // Used for constructing iterators. - void* const* raw_data() const { return rep_ ? rep_->elements : nullptr; } - void** raw_mutable_data() const { - return rep_ ? const_cast(rep_->elements) : nullptr; - } + void* const* raw_data() const { return elements(); } + void** raw_mutable_data() { return elements(); } template typename TypeHandler::Type** mutable_data() { @@ -409,18 +415,20 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { void SwapElements(int index1, int index2) { using std::swap; // enable ADL with fallback - swap(rep_->elements[index1], rep_->elements[index2]); + swap(element_at(index1), element_at(index2)); } template size_t SpaceUsedExcludingSelfLong() const { - size_t allocated_bytes = static_cast(total_size_) * sizeof(void*); - if (rep_ != nullptr) { - for (int i = 0; i < rep_->allocated_size; ++i) { - allocated_bytes += - TypeHandler::SpaceUsedLong(*cast(rep_->elements[i])); - } - allocated_bytes += kRepHeaderSize; + size_t allocated_bytes = + using_sso() + ? 0 + : static_cast(total_size_) * sizeof(void*) + kRepHeaderSize; + const int n = allocated_size(); + void* const* elems = elements(); + for (int i = 0; i < n; ++i) { + allocated_bytes += + TypeHandler::SpaceUsedLong(*cast(elems[i])); } return allocated_bytes; } @@ -430,9 +438,9 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { // Like Add(), but if there are no cleared objects to use, returns nullptr. template typename TypeHandler::Type* AddFromCleared() { - if (rep_ != nullptr && current_size_ < rep_->allocated_size) { + if (current_size_ < allocated_size()) { return cast( - rep_->elements[ExchangeCurrentSize(current_size_ + 1)]); + element_at(ExchangeCurrentSize(current_size_ + 1))); } else { return nullptr; } @@ -447,28 +455,27 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { template void UnsafeArenaAddAllocated(typename TypeHandler::Type* value) { // Make room for the new pointer. - if (!rep_ || current_size_ == total_size_) { + if (current_size_ == total_size_) { // The array is completely full with no cleared objects, so grow it. Reserve(total_size_ + 1); - ++rep_->allocated_size; - } else if (rep_->allocated_size == total_size_) { + ++rep()->allocated_size; + } else if (allocated_size() == total_size_) { // There is no more space in the pointer array because it contains some // cleared objects awaiting reuse. We don't want to grow the array in // this case because otherwise a loop calling AddAllocated() followed by // Clear() would leak memory. - TypeHandler::Delete(cast(rep_->elements[current_size_]), - arena_); - } else if (current_size_ < rep_->allocated_size) { + TypeHandler::Delete(cast(element_at(current_size_)), arena_); + } else if (current_size_ < allocated_size()) { // We have some cleared objects. We don't care about their order, so we // can just move the first one to the end to make space. - rep_->elements[rep_->allocated_size] = rep_->elements[current_size_]; - ++rep_->allocated_size; + element_at(allocated_size()) = element_at(current_size_); + ++rep()->allocated_size; } else { // There are no cleared objects. - ++rep_->allocated_size; + if (!using_sso()) ++rep()->allocated_size; } - rep_->elements[ExchangeCurrentSize(current_size_ + 1)] = value; + element_at(ExchangeCurrentSize(current_size_ + 1)) = value; } template @@ -485,19 +492,21 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { ABSL_DCHECK_GT(current_size_, 0); ExchangeCurrentSize(current_size_ - 1); typename TypeHandler::Type* result = - cast(rep_->elements[current_size_]); - --rep_->allocated_size; - if (current_size_ < rep_->allocated_size) { - // There are cleared elements on the end; replace the removed element - // with the last allocated element. - rep_->elements[current_size_] = rep_->elements[rep_->allocated_size]; + cast(element_at(current_size_)); + if (using_sso()) { + tagged_rep_or_elem_ = nullptr; + } else { + --rep()->allocated_size; + if (current_size_ < allocated_size()) { + // There are cleared elements on the end; replace the removed element + // with the last allocated element. + element_at(current_size_) = element_at(allocated_size()); + } } return result; } - int ClearedCount() const { - return rep_ ? (rep_->allocated_size - current_size_) : 0; - } + int ClearedCount() const { return allocated_size() - current_size_; } template void AddCleared(typename TypeHandler::Type* value) { @@ -506,10 +515,14 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { "RepeatedPtrField not on an arena."; ABSL_DCHECK(TypeHandler::GetOwningArena(value) == nullptr) << "AddCleared() can only accept values not on an arena."; - if (!rep_ || rep_->allocated_size == total_size_) { + if (allocated_size() == total_size_) { Reserve(total_size_ + 1); } - rep_->elements[rep_->allocated_size++] = value; + if (using_sso()) { + tagged_rep_or_elem_ = value; + } else { + element_at(rep()->allocated_size++) = value; + } } template @@ -517,9 +530,16 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { ABSL_DCHECK(GetOwningArena() == nullptr) << "ReleaseCleared() can only be used on a RepeatedPtrField not on " << "an arena."; - ABSL_DCHECK(rep_ != nullptr); - ABSL_DCHECK_GT(rep_->allocated_size, current_size_); - return cast(rep_->elements[--rep_->allocated_size]); + ABSL_DCHECK(tagged_rep_or_elem_ != nullptr); + ABSL_DCHECK_GT(allocated_size(), current_size_); + if (using_sso()) { + auto* result = + reinterpret_cast(tagged_rep_or_elem_); + tagged_rep_or_elem_ = nullptr; + return result; + } else { + return cast(element_at(--rep()->allocated_size)); + } } template @@ -528,18 +548,18 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { Arena* element_arena = reinterpret_cast(TypeHandler::GetOwningArena(value)); Arena* arena = GetOwningArena(); - if (arena == element_arena && rep_ && rep_->allocated_size < total_size_) { + if (arena == element_arena && allocated_size() < total_size_) { // Fast path: underlying arena representation (tagged pointer) is equal to // our arena pointer, and we can add to array without resizing it (at // least one slot that is not allocated). - void** elems = rep_->elements; - if (current_size_ < rep_->allocated_size) { + void** elems = elements(); + if (current_size_ < allocated_size()) { // Make space at [current] by moving first allocated element to end of // allocated list. - elems[rep_->allocated_size] = elems[current_size_]; + elems[allocated_size()] = elems[current_size_]; } elems[ExchangeCurrentSize(current_size_ + 1)] = value; - rep_->allocated_size = rep_->allocated_size + 1; + if (!using_sso()) ++rep()->allocated_size; } else { AddAllocatedSlowWithCopy(value, element_arena, arena); } @@ -550,18 +570,18 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { // AddAllocated version that does not implement arena-safe copying // behavior. typename TypeHandler::Type* value, std::false_type) { - if (rep_ && rep_->allocated_size < total_size_) { + if (allocated_size() < total_size_) { // Fast path: underlying arena representation (tagged pointer) is equal to // our arena pointer, and we can add to array without resizing it (at // least one slot that is not allocated). - void** elems = rep_->elements; - if (current_size_ < rep_->allocated_size) { + void** elems = elements(); + if (current_size_ < allocated_size()) { // Make space at [current] by moving first allocated element to end of // allocated list. - elems[rep_->allocated_size] = elems[current_size_]; + elems[allocated_size()] = elems[current_size_]; } elems[ExchangeCurrentSize(current_size_ + 1)] = value; - ++rep_->allocated_size; + if (!using_sso()) ++rep()->allocated_size; } else { UnsafeArenaAddAllocated(value); } @@ -685,7 +705,41 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { sizeof(void*)]; }; static constexpr size_t kRepHeaderSize = offsetof(Rep, elements); - Rep* rep_; + void* const* elements() const { + return using_sso() ? &tagged_rep_or_elem_ : +rep()->elements; + } + void** elements() { + return using_sso() ? &tagged_rep_or_elem_ : +rep()->elements; + } + + void*& element_at(int index) { + if (using_sso()) { + ABSL_DCHECK_EQ(index, 0); + return tagged_rep_or_elem_; + } + return rep()->elements[index]; + } + const void* element_at(int index) const { + return const_cast(this)->element_at(index); + } + + int allocated_size() const { + return using_sso() ? (tagged_rep_or_elem_ != nullptr ? 1 : 0) + : rep()->allocated_size; + } + Rep* rep() { + ABSL_DCHECK(!using_sso()); + return reinterpret_cast( + reinterpret_cast(tagged_rep_or_elem_) - 1); + } + const Rep* rep() const { + return const_cast(this)->rep(); + } + + bool using_sso() const { + return (reinterpret_cast(tagged_rep_or_elem_) & 1) == 0; + } + void* tagged_rep_or_elem_; template static inline typename TypeHandler::Type* cast(void* element) { @@ -701,13 +755,13 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { template PROTOBUF_NOINLINE void ClearNonEmpty() { const int n = current_size_; - void* const* elements = rep_->elements; + void* const* elems = elements(); int i = 0; ABSL_DCHECK_GT( n, 0); // do/while loop to avoid initial test because we know n > 0 do { - TypeHandler::Clear(cast(elements[i++])); + TypeHandler::Clear(cast(elems[i++])); } while (i < n); ExchangeCurrentSize(0); } @@ -716,28 +770,29 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { // pointer to the type-specific (templated) inner allocate/merge loop. PROTOBUF_NOINLINE void MergeFromInternal( const RepeatedPtrFieldBase& other, - void (RepeatedPtrFieldBase::*inner_loop)(void**, void**, int, int)) { + void (RepeatedPtrFieldBase::*inner_loop)(void**, void* const*, int, + int)) { // Note: wrapper has already guaranteed that other.rep_ != nullptr here. int other_size = other.current_size_; - void** other_elements = other.rep_->elements; + void* const* other_elements = other.elements(); void** new_elements = InternalExtend(other_size); - int allocated_elems = rep_->allocated_size - current_size_; + int allocated_elems = allocated_size() - current_size_; (this->*inner_loop)(new_elements, other_elements, other_size, allocated_elems); ExchangeCurrentSize(current_size_ + other_size); - if (rep_->allocated_size < current_size_) { - rep_->allocated_size = current_size_; + if (allocated_size() < current_size_) { + rep()->allocated_size = current_size_; } } // Merges other_elems to our_elems. template PROTOBUF_NOINLINE void MergeFromInnerLoop(void** our_elems, - void** other_elems, int length, - int already_allocated) { + void* const* other_elems, + int length, int already_allocated) { if (already_allocated < length) { Arena* arena = GetOwningArena(); - typename TypeHandler::Type* elem_prototype = + auto* elem_prototype = reinterpret_cast(other_elems[0]); for (int i = already_allocated; i < length; i++) { // Allocate a new empty element that we'll merge into below