RepeatedPtrField clears elements lazily.

This allows to reduce binary bloat (all calls to Clear are now defined out of line) and eliminate redundant calls to clear (since we only call it when it is needed).

PiperOrigin-RevId: 590944438
pull/15091/head
Protobuf Team Bot 1 year ago committed by Copybara-Service
parent f0a8c474cc
commit 1db8ed47c2
  1. 15
      src/google/protobuf/generated_message_reflection.cc
  2. 2
      src/google/protobuf/implicit_weak_message.h
  3. 1
      src/google/protobuf/repeated_field_unittest.cc
  4. 50
      src/google/protobuf/repeated_ptr_field.cc
  5. 203
      src/google/protobuf/repeated_ptr_field.h

@ -1431,7 +1431,7 @@ void Reflection::ClearField(Message* message,
switch (field->options().ctype()) { switch (field->options().ctype()) {
default: // TODO: Support other string reps. default: // TODO: Support other string reps.
case FieldOptions::STRING: case FieldOptions::STRING:
MutableRaw<RepeatedPtrField<std::string> >(message, field)->Clear(); MutableRaw<RepeatedPtrFieldBase>(message, field)->Clear();
break; break;
} }
break; break;
@ -1441,10 +1441,7 @@ void Reflection::ClearField(Message* message,
if (IsMapFieldInApi(field)) { if (IsMapFieldInApi(field)) {
MutableRaw<MapFieldBase>(message, field)->Clear(); MutableRaw<MapFieldBase>(message, field)->Clear();
} else { } else {
// We don't know which subclass of RepeatedPtrFieldBase the type is, MutableRaw<RepeatedPtrFieldBase>(message, field)->Clear();
// so we use RepeatedPtrFieldBase directly.
MutableRaw<RepeatedPtrFieldBase>(message, field)
->Clear<GenericTypeHandler<Message> >();
} }
break; break;
} }
@ -1481,8 +1478,7 @@ void Reflection::RemoveLast(Message* message,
switch (field->options().ctype()) { switch (field->options().ctype()) {
default: // TODO: Support other string reps. default: // TODO: Support other string reps.
case FieldOptions::STRING: case FieldOptions::STRING:
MutableRaw<RepeatedPtrField<std::string> >(message, field) MutableRaw<RepeatedPtrFieldBase>(message, field)->RemoveLast();
->RemoveLast();
break; break;
} }
break; break;
@ -1491,10 +1487,9 @@ void Reflection::RemoveLast(Message* message,
if (IsMapFieldInApi(field)) { if (IsMapFieldInApi(field)) {
MutableRaw<MapFieldBase>(message, field) MutableRaw<MapFieldBase>(message, field)
->MutableRepeatedField() ->MutableRepeatedField()
->RemoveLast<GenericTypeHandler<Message> >(); ->RemoveLast();
} else { } else {
MutableRaw<RepeatedPtrFieldBase>(message, field) MutableRaw<RepeatedPtrFieldBase>(message, field)->RemoveLast();
->RemoveLast<GenericTypeHandler<Message> >();
} }
break; break;
} }

@ -201,7 +201,7 @@ struct WeakRepeatedPtrField {
} }
T* Add() { return weak.Add(); } T* Add() { return weak.Add(); }
void Clear() { base().template Clear<TypeHandler>(); } void Clear() { base().Clear(); }
void MergeFrom(const WeakRepeatedPtrField& other) { void MergeFrom(const WeakRepeatedPtrField& other) {
if (other.empty()) return; if (other.empty()) return;
base().template MergeFrom<MessageLite>(other.base()); base().template MergeFrom<MessageLite>(other.base());

@ -1658,7 +1658,6 @@ TEST(RepeatedPtrField, ClearedElements) {
EXPECT_EQ(field.ClearedCount(), 0); EXPECT_EQ(field.ClearedCount(), 0);
field.RemoveLast(); field.RemoveLast();
EXPECT_TRUE(original->empty());
EXPECT_EQ(field.ClearedCount(), 1); EXPECT_EQ(field.ClearedCount(), 1);
EXPECT_EQ(field.Add(), EXPECT_EQ(field.Add(),

@ -54,7 +54,7 @@ void** RepeatedPtrFieldBase::InternalExtend(int extend_amount) {
new_rep = reinterpret_cast<Rep*>(Arena::CreateArray<char>(arena, bytes)); new_rep = reinterpret_cast<Rep*>(Arena::CreateArray<char>(arena, bytes));
} }
if (using_sso()) { if (using_element()) {
new_rep->allocated_size = tagged_rep_or_elem_ != nullptr ? 1 : 0; new_rep->allocated_size = tagged_rep_or_elem_ != nullptr ? 1 : 0;
new_rep->elements[0] = tagged_rep_or_elem_; new_rep->elements[0] = tagged_rep_or_elem_;
} else { } else {
@ -75,7 +75,7 @@ void** RepeatedPtrFieldBase::InternalExtend(int extend_amount) {
tagged_rep_or_elem_ = tagged_rep_or_elem_ =
reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(new_rep) + 1); reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(new_rep) + 1);
capacity_proxy_ = new_capacity - kSSOCapacity; capacity_proxy_ = new_capacity - kInlinedCapacity;
return &new_rep->elements[current_size_]; return &new_rep->elements[current_size_];
} }
@ -94,18 +94,31 @@ void RepeatedPtrFieldBase::DestroyProtos() {
tagged_rep_or_elem_ = nullptr; tagged_rep_or_elem_ = nullptr;
} }
template <typename F> namespace {
auto* RepeatedPtrFieldBase::AddInternal(F factory) { template <typename T>
struct ElementRecycler {
static void clear(void* p) { static_cast<T*>(p)->Clear(); }
};
template <>
struct ElementRecycler<std::string> {
static void clear(void* str) { static_cast<std::string*>(str)->clear(); }
};
} // namespace
template <typename Recycler, typename Factory>
void* RepeatedPtrFieldBase::AddInternal(Factory factory) {
Arena* const arena = GetArena(); Arena* const arena = GetArena();
using Result = decltype(factory(arena));
if (tagged_rep_or_elem_ == nullptr) { if (tagged_rep_or_elem_ == nullptr) {
ExchangeCurrentSize(1); ExchangeCurrentSize(1);
tagged_rep_or_elem_ = factory(arena); tagged_rep_or_elem_ = factory(arena);
return static_cast<Result>(tagged_rep_or_elem_); return tagged_rep_or_elem_;
} }
if (using_sso()) { if (using_element()) {
if (ExchangeCurrentSize(1) == 0) { if (ExchangeCurrentSize(1) == 0) {
return static_cast<Result>(tagged_rep_or_elem_); Recycler::clear(tagged_rep_or_elem_);
return tagged_rep_or_elem_;
} }
} else { } else {
absl::PrefetchToLocalCache(rep()); absl::PrefetchToLocalCache(rep());
@ -115,23 +128,28 @@ auto* RepeatedPtrFieldBase::AddInternal(F factory) {
} else { } else {
Rep* r = rep(); Rep* r = rep();
if (current_size_ != r->allocated_size) { if (current_size_ != r->allocated_size) {
return static_cast<Result>( void* cached = r->elements[ExchangeCurrentSize(current_size_ + 1)];
r->elements[ExchangeCurrentSize(current_size_ + 1)]); Recycler::clear(cached);
return cached;
} }
} }
Rep* r = rep(); Rep* r = rep();
++r->allocated_size; ++r->allocated_size;
void*& result = r->elements[ExchangeCurrentSize(current_size_ + 1)]; void*& result = r->elements[ExchangeCurrentSize(current_size_ + 1)];
result = factory(arena); result = factory(arena);
return static_cast<Result>(result); return result;
}
void* RepeatedPtrFieldBase::AddMessageLite(ElementFactory factory) {
return AddInternal<ElementRecycler<MessageLite>>(factory);
} }
void* RepeatedPtrFieldBase::AddOutOfLineHelper(ElementFactory factory) { void* RepeatedPtrFieldBase::AddString() {
return AddInternal(factory); return AddInternal<ElementRecycler<std::string>>(NewStringElement);
} }
void RepeatedPtrFieldBase::CloseGap(int start, int num) { void RepeatedPtrFieldBase::CloseGap(int start, int num) {
if (using_sso()) { if (using_element()) {
if (start == 0 && num == 1) { if (start == 0 && num == 1) {
tagged_rep_or_elem_ = nullptr; tagged_rep_or_elem_ = nullptr;
} }
@ -146,7 +164,8 @@ void RepeatedPtrFieldBase::CloseGap(int start, int num) {
} }
MessageLite* RepeatedPtrFieldBase::AddMessage(const MessageLite* prototype) { MessageLite* RepeatedPtrFieldBase::AddMessage(const MessageLite* prototype) {
return AddInternal([prototype](Arena* a) { return prototype->New(a); }); return static_cast<MessageLite*>(AddInternal<ElementRecycler<MessageLite>>(
[prototype](Arena* a) { return prototype->New(a); }));
} }
void InternalOutOfLineDeleteMessageLite(MessageLite* message) { void InternalOutOfLineDeleteMessageLite(MessageLite* message) {
@ -197,6 +216,7 @@ int RepeatedPtrFieldBase::MergeIntoClearedMessages(
ABSL_DCHECK(typeid(*src[i]) == typeid(*src[0])) ABSL_DCHECK(typeid(*src[i]) == typeid(*src[0]))
<< typeid(*src[i]).name() << " vs " << typeid(*src[0]).name(); << typeid(*src[i]).name() << " vs " << typeid(*src[0]).name();
#endif #endif
dst[i]->Clear();
dst[i]->CheckTypeAndMergeFrom(*src[i]); dst[i]->CheckTypeAndMergeFrom(*src[i]);
} }
return count; return count;

@ -116,7 +116,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
template <typename Handler> template <typename Handler>
using Value = typename Handler::Type; using Value = typename Handler::Type;
static constexpr int kSSOCapacity = 1; static constexpr int kInlinedCapacity = 1;
using ElementFactory = void* (*)(Arena*); using ElementFactory = void* (*)(Arena*);
@ -158,7 +158,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
// //
// * prefer `SizeAtCapacity()` to `size() == Capacity()`; // * prefer `SizeAtCapacity()` to `size() == Capacity()`;
// * prefer `AllocatedSizeAtCapacity()` to `allocated_size() == Capacity()`. // * prefer `AllocatedSizeAtCapacity()` to `allocated_size() == Capacity()`.
int Capacity() const { return capacity_proxy_ + kSSOCapacity; } int Capacity() const { return capacity_proxy_ + kInlinedCapacity; }
template <typename TypeHandler> template <typename TypeHandler>
const Value<TypeHandler>& at(int index) const { const Value<TypeHandler>& at(int index) const {
@ -183,7 +183,10 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
template <typename Handler> template <typename Handler>
Value<Handler>* Add() { Value<Handler>* Add() {
return cast<Handler>(AddOutOfLineHelper(Handler::GetNewFunc())); if (std::is_same<Value<Handler>, std::string>{}) {
return cast<Handler>(AddString());
}
return cast<Handler>(AddMessageLite(Handler::GetNewFunc()));
} }
template < template <
@ -196,7 +199,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
return; return;
} }
MaybeExtend(); MaybeExtend();
if (!using_sso()) ++rep()->allocated_size; if (!using_element()) ++rep()->allocated_size;
auto* result = TypeHandler::New(arena_, std::move(value)); auto* result = TypeHandler::New(arena_, std::move(value));
element_at(ExchangeCurrentSize(current_size_ + 1)) = result; element_at(ExchangeCurrentSize(current_size_ + 1)) = result;
} }
@ -218,15 +221,14 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
Delete<H>(elems[i], nullptr); Delete<H>(elems[i], nullptr);
} }
if (!using_sso()) { if (!using_element()) {
internal::SizedDelete(rep(), internal::SizedDelete(rep(),
Capacity() * sizeof(elems[0]) + kRepHeaderSize); Capacity() * sizeof(elems[0]) + kRepHeaderSize);
} }
} }
inline bool NeedsDestroy() const { inline bool NeedsDestroy() const {
// Either there is an allocated element in SSO buffer or there is an // tagged_rep_or_elem_ contains either allocated element or allocated `Rep`.
// allocated Rep.
return tagged_rep_or_elem_ != nullptr; return tagged_rep_or_elem_ != nullptr;
} }
void DestroyProtos(); void DestroyProtos();
@ -249,15 +251,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
// Pre-condition: prototype must not be nullptr. // Pre-condition: prototype must not be nullptr.
MessageLite* AddMessage(const MessageLite* prototype); MessageLite* AddMessage(const MessageLite* prototype);
template <typename TypeHandler> void Clear() { ExchangeCurrentSize(0); }
void Clear() {
const int n = current_size_;
ABSL_DCHECK_GE(n, 0);
if (n > 0) {
using H = CommonHandler<TypeHandler>;
ClearNonEmpty<H>();
}
}
// Appends all message values from `from` to this instance. // Appends all message values from `from` to this instance.
template <typename T> template <typename T>
@ -294,24 +288,21 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
ABSL_DCHECK_EQ(current_size_, allocated_size()); ABSL_DCHECK_EQ(current_size_, allocated_size());
MaybeExtend(); MaybeExtend();
element_at(current_size_++) = value; element_at(current_size_++) = value;
if (!using_sso()) ++rep()->allocated_size; if (!using_element()) ++rep()->allocated_size;
} }
protected: protected:
template <typename TypeHandler>
void RemoveLast() { void RemoveLast() {
ABSL_DCHECK_GT(current_size_, 0); ABSL_DCHECK_GT(current_size_, 0);
ExchangeCurrentSize(current_size_ - 1); ExchangeCurrentSize(current_size_ - 1);
using H = CommonHandler<TypeHandler>;
H::Clear(cast<H>(element_at(current_size_)));
} }
template <typename TypeHandler> template <typename TypeHandler>
void CopyFrom(const RepeatedPtrFieldBase& other) { void CopyFrom(const RepeatedPtrFieldBase& other) {
if (&other == this) return; if (&other == this) return;
RepeatedPtrFieldBase::Clear<TypeHandler>(); RepeatedPtrFieldBase::Clear();
if (other.empty()) return; if (other.empty()) return;
RepeatedPtrFieldBase::MergeFrom<typename TypeHandler::Type>(other); RepeatedPtrFieldBase::MergeFrom<Value<TypeHandler>>(other);
} }
void CloseGap(int start, int num); void CloseGap(int start, int num);
@ -366,7 +357,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
template <typename TypeHandler> template <typename TypeHandler>
PROTOBUF_NOINLINE size_t SpaceUsedExcludingSelfLong() const { PROTOBUF_NOINLINE size_t SpaceUsedExcludingSelfLong() const {
size_t allocated_bytes = size_t allocated_bytes =
using_sso() using_element()
? 0 ? 0
: static_cast<size_t>(Capacity()) * sizeof(void*) + kRepHeaderSize; : static_cast<size_t>(Capacity()) * sizeof(void*) + kRepHeaderSize;
const int n = allocated_size(); const int n = allocated_size();
@ -380,15 +371,15 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
// Advanced memory management -------------------------------------- // Advanced memory management --------------------------------------
// Like Add(), but if there are no cleared objects to use, returns nullptr. // Returns a pointer to a cleared object ready to reuse if there is a spare
// allocated object or nullptr otherwise.
template <typename TypeHandler> template <typename TypeHandler>
Value<TypeHandler>* AddFromCleared() { Value<TypeHandler>* AddFromCleared() {
if (current_size_ < allocated_size()) { if (ClearedCount() == 0) return nullptr;
return cast<TypeHandler>( auto* value =
element_at(ExchangeCurrentSize(current_size_ + 1))); cast<TypeHandler>(element_at(ExchangeCurrentSize(current_size_ + 1)));
} else { CommonHandler<TypeHandler>::Clear(value);
return nullptr; return value;
}
} }
template <typename TypeHandler> template <typename TypeHandler>
@ -410,7 +401,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
elems[allocated_size()] = elems[current_size_]; elems[allocated_size()] = elems[current_size_];
} }
elems[ExchangeCurrentSize(current_size_ + 1)] = value; elems[ExchangeCurrentSize(current_size_ + 1)] = value;
if (!using_sso()) ++rep()->allocated_size; if (!using_element()) ++rep()->allocated_size;
} }
template <typename TypeHandler> template <typename TypeHandler>
@ -418,24 +409,24 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
ABSL_DCHECK_NE(value, nullptr); ABSL_DCHECK_NE(value, nullptr);
// Make room for the new pointer. // Make room for the new pointer.
if (SizeAtCapacity()) { if (SizeAtCapacity()) {
// The array is completely full with no cleared objects, so grow it. // The array is completely full, so grow it.
InternalExtend(1); InternalExtend(1);
++rep()->allocated_size; ++rep()->allocated_size;
} else if (AllocatedSizeAtCapacity()) { } else if (AllocatedSizeAtCapacity()) {
// There is no more space in the pointer array because it contains some // There is no more space in the pointer array because it contains some
// cleared objects awaiting reuse. We don't want to grow the array in // objects awaiting reuse. We don't want to grow the array in
// this case because otherwise a loop calling AddAllocated() followed by // this case because otherwise a loop calling AddAllocated() followed by
// Clear() would leak memory. // Clear() would leak memory.
using H = CommonHandler<TypeHandler>; using H = CommonHandler<TypeHandler>;
Delete<H>(element_at(current_size_), arena_); Delete<H>(element_at(current_size_), arena_);
} else if (current_size_ < allocated_size()) { } else if (current_size_ < allocated_size()) {
// We have some cleared objects. We don't care about their order, so we // We have some unused allocated objects. Their order is not important,
// can just move the first one to the end to make space. // so we move the first one to the end to make room for the pointer.
element_at(allocated_size()) = element_at(current_size_); element_at(allocated_size()) = element_at(current_size_);
++rep()->allocated_size; ++rep()->allocated_size;
} else { } else {
// There are no cleared objects. // There are no unused allocated objects.
if (!using_sso()) ++rep()->allocated_size; if (!using_element()) ++rep()->allocated_size;
} }
element_at(ExchangeCurrentSize(current_size_ + 1)) = value; element_at(ExchangeCurrentSize(current_size_ + 1)) = value;
@ -464,13 +455,13 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
ABSL_DCHECK_GT(current_size_, 0); ABSL_DCHECK_GT(current_size_, 0);
ExchangeCurrentSize(current_size_ - 1); ExchangeCurrentSize(current_size_ - 1);
auto* result = cast<TypeHandler>(element_at(current_size_)); auto* result = cast<TypeHandler>(element_at(current_size_));
if (using_sso()) { if (using_element()) {
tagged_rep_or_elem_ = nullptr; tagged_rep_or_elem_ = nullptr;
} else { } else {
--rep()->allocated_size; --rep()->allocated_size;
if (current_size_ < allocated_size()) { if (current_size_ < allocated_size()) {
// There are cleared elements on the end; replace the removed element // There are unused allocated elements on the end; replace the removed
// with the last allocated element. // element with the last allocated element.
element_at(current_size_) = element_at(allocated_size()); element_at(current_size_) = element_at(allocated_size());
} }
} }
@ -486,7 +477,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
ABSL_DCHECK(TypeHandler::GetArena(value) == nullptr) ABSL_DCHECK(TypeHandler::GetArena(value) == nullptr)
<< "AddCleared() can only accept values not on an arena."; << "AddCleared() can only accept values not on an arena.";
MaybeExtend(); MaybeExtend();
if (using_sso()) { if (using_element()) {
tagged_rep_or_elem_ = value; tagged_rep_or_elem_ = value;
} else { } else {
element_at(rep()->allocated_size++) = value; element_at(rep()->allocated_size++) = value;
@ -500,13 +491,14 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
<< "an arena."; << "an arena.";
ABSL_DCHECK(tagged_rep_or_elem_ != nullptr); ABSL_DCHECK(tagged_rep_or_elem_ != nullptr);
ABSL_DCHECK_GT(allocated_size(), current_size_); ABSL_DCHECK_GT(allocated_size(), current_size_);
if (using_sso()) { void* result;
auto* result = cast<TypeHandler>(tagged_rep_or_elem_); if (using_element()) {
tagged_rep_or_elem_ = nullptr; result = std::exchange(tagged_rep_or_elem_, nullptr);
return result;
} else { } else {
return cast<TypeHandler>(element_at(--rep()->allocated_size)); result = element_at(--rep()->allocated_size);
} }
TypeHandler::Clear(cast<TypeHandler>(result));
return cast<TypeHandler>(result);
} }
// Slowpath handles all cases, copying if necessary. // Slowpath handles all cases, copying if necessary.
@ -544,7 +536,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
// than three times. // than three times.
RepeatedPtrFieldBase temp(other->GetArena()); RepeatedPtrFieldBase temp(other->GetArena());
if (!this->empty()) { if (!this->empty()) {
temp.MergeFrom<typename TypeHandler::Type>(*this); temp.MergeFrom<Value<TypeHandler>>(*this);
} }
this->CopyFrom<TypeHandler>(*other); this->CopyFrom<TypeHandler>(*other);
other->InternalSwap(&temp); other->InternalSwap(&temp);
@ -625,30 +617,29 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
ABSL_DCHECK_LE(size(), allocated_size()); ABSL_DCHECK_LE(size(), allocated_size());
ABSL_DCHECK_LE(allocated_size(), Capacity()); ABSL_DCHECK_LE(allocated_size(), Capacity());
// This is equivalent to `current_size_ == Capacity()`. // This is equivalent to `current_size_ == Capacity()`.
// Assuming `Capacity()` function is inlined, compiler is likely to optimize //
// away "+ kSSOCapacity" and reduce it to "current_size_ > capacity_proxy_" // Using less than instead of equality gives compiler an opportunity to
// which is an instruction less than "current_size_ == capacity_proxy_ + 1". // generate less instructions.
return current_size_ >= Capacity(); return current_size_ >= Capacity();
} }
inline bool AllocatedSizeAtCapacity() const { inline bool AllocatedSizeAtCapacity() const {
// Harden invariant size() <= allocated_size() <= Capacity(). // Harden invariant size() <= allocated_size() <= Capacity().
ABSL_DCHECK_LE(size(), allocated_size()); ABSL_DCHECK_LE(size(), allocated_size());
ABSL_DCHECK_LE(allocated_size(), Capacity()); ABSL_DCHECK_LE(allocated_size(), Capacity());
// This combines optimization mentioned in `SizeAtCapacity()` and simplifies // See comment in SizeAtCapacity().
// `allocated_size()` in sso case. return using_element() ? (tagged_rep_or_elem_ != nullptr)
return using_sso() ? (tagged_rep_or_elem_ != nullptr)
: rep()->allocated_size >= Capacity(); : rep()->allocated_size >= Capacity();
} }
void* const* elements() const { void* const* elements() const {
return using_sso() ? &tagged_rep_or_elem_ : +rep()->elements; return using_element() ? &tagged_rep_or_elem_ : +rep()->elements;
} }
void** elements() { void** elements() {
return using_sso() ? &tagged_rep_or_elem_ : +rep()->elements; return using_element() ? &tagged_rep_or_elem_ : +rep()->elements;
} }
void*& element_at(int index) { void*& element_at(int index) {
if (using_sso()) { if (using_element()) {
ABSL_DCHECK_EQ(index, 0); ABSL_DCHECK_EQ(index, 0);
return tagged_rep_or_elem_; return tagged_rep_or_elem_;
} }
@ -659,11 +650,11 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
} }
int allocated_size() const { int allocated_size() const {
return using_sso() ? (tagged_rep_or_elem_ != nullptr ? 1 : 0) return using_element() ? (tagged_rep_or_elem_ != nullptr ? 1 : 0)
: rep()->allocated_size; : rep()->allocated_size;
} }
Rep* rep() { Rep* rep() {
ABSL_DCHECK(!using_sso()); ABSL_DCHECK(!using_element());
return reinterpret_cast<Rep*>( return reinterpret_cast<Rep*>(
reinterpret_cast<uintptr_t>(tagged_rep_or_elem_) - 1); reinterpret_cast<uintptr_t>(tagged_rep_or_elem_) - 1);
} }
@ -671,7 +662,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
return const_cast<RepeatedPtrFieldBase*>(this)->rep(); return const_cast<RepeatedPtrFieldBase*>(this)->rep();
} }
bool using_sso() const { bool using_element() const {
return (reinterpret_cast<uintptr_t>(tagged_rep_or_elem_) & 1) == 0; return (reinterpret_cast<uintptr_t>(tagged_rep_or_elem_) & 1) == 0;
} }
@ -688,28 +679,13 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
TypeHandler::Delete(cast<TypeHandler>(obj), arena); TypeHandler::Delete(cast<TypeHandler>(obj), arena);
} }
// Out-of-line helper routine for Clear() once the inlined check has // Merges messages from `from` into available, allocated messages sitting in
// determined the container is non-empty // the range `[size(), allocated_size())`. Returns the number of message
template <typename TypeHandler> // merged which is `ClearedCount(), from.size())`.
PROTOBUF_NOINLINE void ClearNonEmpty() { // Note that this function does explicitly NOT update `current_size_`. This
const int n = current_size_; // function is out of line as it should be the slow path: this scenario only
void* const* elems = elements(); // happens when a caller constructs and fills a repeated field, then shrinks
int i = 0; // it, and then merges additional messages into it.
ABSL_DCHECK_GT(n, 0);
// do/while loop to avoid initial test because we know n > 0
do {
TypeHandler::Clear(cast<TypeHandler>(elems[i++]));
} while (i < n);
ExchangeCurrentSize(0);
}
// Merges messages from `from` into available, cleared messages sitting in the
// range `[size(), allocated_size())`. Returns the number of message merged
// which is `ClearedCount(), from.size())`.
// Note that this function does explicitly NOT update `current_size_`.
// This function is out of line as it should be the slow path: this scenario
// only happens when a caller constructs and fills a repeated field, then
// shrinks it, and then merges additional messages into it.
int MergeIntoClearedMessages(const RepeatedPtrFieldBase& from); int MergeIntoClearedMessages(const RepeatedPtrFieldBase& from);
// Appends all messages from `from` to this instance, using the // Appends all messages from `from` to this instance, using the
@ -736,40 +712,55 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase {
// Returns a pointer to the element directly beyond the last element. // Returns a pointer to the element directly beyond the last element.
inline void** InternalReserve(int n) { inline void** InternalReserve(int n) {
if (n <= Capacity()) { if (n <= Capacity()) {
void** elements = using_sso() ? &tagged_rep_or_elem_ : rep()->elements; return elements() + current_size_;
return elements + current_size_;
} }
return InternalExtend(n - Capacity()); return InternalExtend(n - Capacity());
} }
// Internal helper for Add that keeps definition out-of-line. // Internal helpers for Add that keep definition out-of-line.
void* AddOutOfLineHelper(ElementFactory factory); void* AddMessageLite(ElementFactory factory);
void* AddString();
// Common implementation used by various Add* methods. `factory` is an object // Common implementation used by various Add* methods. `factory` is an object
// used to construct a new element unless there are spare cleared elements // used to construct a new element unless there are spare allocated elements
// ready for reuse. Returns pointer to the new element. // ready for reuse. Returns pointer to the new element.
// //
// Note: avoid inlining this function in methods such as `Add()` as this would // Note: avoid inlining this function in methods such as `Add()` as this would
// drastically increase binary size due to template instantiation and implicit // drastically increase binary size due to template instantiation and implicit
// inlining. Instead, use wrapper functions with out-of-line definition // inlining.
// similar to `AddOutOfLineHelper`. template <typename Recycler, typename Factory>
template <typename F> void* AddInternal(Factory factory);
auto* AddInternal(F factory);
// A few notes on internal representation: // A few notes on internal representation:
// //
// We use an indirected approach, with struct Rep, to keep // * Class layout is optimized to minimize the size: 24 bytes on x86-64.
// sizeof(RepeatedPtrFieldBase) equivalent to what it was before arena support // * The elements can be stored in one of the two ways and `using_element()`
// was added; namely, 3 8-byte machine words on x86-64. An instance of Rep is // tells which one is currently used.
// allocated only when the repeated field is non-empty, and it is a //
// dynamically-sized struct (the header is directly followed by elements[]). // In case of using_element():
// We place arena_ and current_size_ directly in the object to avoid cache //
// misses due to the indirection, because these fields are checked frequently. // tagged_rep_or_elem_ is a storage for at most one pointer.
// Placing all fields directly in the RepeatedPtrFieldBase instance would cost // Number of allocated objects (0 or 1) is determined whether
// significant performance for memory-sensitive workloads. // tagged_rep_or_elem_ is nullptr.
//
// Otherwise,
//
// tagged_rep_or_elem_ is tagged (LSB is 1) pointer to `Rep`, where
// `Rep` contains number of allocated objects as well as the buffer with
// pointers to allocated elements. Rep allows to (a) keep the sizeof small
// (b) allocate both buffer for elements and an integer with allocated
// objects count in one shot.
//
// In both cases, RepeatedPtrFieldBase may own allocated but unused objects:
//
// 1. Their count is determined by `ClearedCount()`.
// 2. Pointers to them are stored directly after pointers to used objects.
// 3. They can be reused in order to avoid extra allocation (note that in
// some cases these objects need to be cleared with `TypeHandler::Clear`
// before they can be reused).
void* tagged_rep_or_elem_; void* tagged_rep_or_elem_;
int current_size_; int current_size_;
int capacity_proxy_; // we store `capacity - kSSOCapacity` as an optimization int capacity_proxy_; // See `Capacity()`
Arena* arena_; Arena* arena_;
}; };
@ -984,7 +975,7 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase {
pointer Mutable(int index) ABSL_ATTRIBUTE_LIFETIME_BOUND; pointer Mutable(int index) ABSL_ATTRIBUTE_LIFETIME_BOUND;
// Unlike std::vector, adding an element to a RepeatedPtrField doesn't always // Unlike std::vector, adding an element to a RepeatedPtrField doesn't always
// make a new element; it might re-use an element left over from when the // make a new element; it might reuse an element left over from when the
// field was Clear()'d or resize()'d smaller. For this reason, Add() is the // field was Clear()'d or resize()'d smaller. For this reason, Add() is the
// fastest API for adding a new element. // fastest API for adding a new element.
pointer Add() ABSL_ATTRIBUTE_LIFETIME_BOUND; pointer Add() ABSL_ATTRIBUTE_LIFETIME_BOUND;
@ -1166,9 +1157,9 @@ class RepeatedPtrField final : private internal::RepeatedPtrFieldBase {
void UnsafeArenaExtractSubrange(int start, int num, Element** elements); void UnsafeArenaExtractSubrange(int start, int num, Element** elements);
// When elements are removed by calls to RemoveLast() or Clear(), they // When elements are removed by calls to RemoveLast() or Clear(), they
// are not actually freed. Instead, they are cleared and kept so that // are not actually freed. Instead, they are kept so that they can be reused
// they can be reused later. This can save lots of CPU time when // later. This can save lots of CPU time when repeatedly reusing a protocol
// repeatedly reusing a protocol message for similar purposes. // message for similar purposes.
// //
// Hardcore programs may choose to manipulate these cleared objects // Hardcore programs may choose to manipulate these cleared objects
// to better optimize memory management using the following routines. // to better optimize memory management using the following routines.
@ -1384,7 +1375,7 @@ inline void RepeatedPtrField<Element>::Add(Iter begin, Iter end) {
template <typename Element> template <typename Element>
inline void RepeatedPtrField<Element>::RemoveLast() { inline void RepeatedPtrField<Element>::RemoveLast() {
RepeatedPtrFieldBase::RemoveLast<TypeHandler>(); RepeatedPtrFieldBase::RemoveLast();
} }
template <typename Element> template <typename Element>
@ -1459,7 +1450,7 @@ inline void RepeatedPtrField<Element>::UnsafeArenaExtractSubrange(
template <typename Element> template <typename Element>
inline void RepeatedPtrField<Element>::Clear() { inline void RepeatedPtrField<Element>::Clear() {
RepeatedPtrFieldBase::Clear<TypeHandler>(); RepeatedPtrFieldBase::Clear();
} }
template <typename Element> template <typename Element>

Loading…
Cancel
Save