diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 30a3382d87..d986479ef7 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -242,6 +242,27 @@ cc_library( ], ) +cc_library( + name = "string_block", + hdrs = ["string_block.h"], + include_prefix = "google/protobuf", + deps = [ + ":arena_align", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + ], +) + +cc_test( + name = "string_block_test", + srcs = ["string_block_test.cc"], + deps = [ + ":string_block", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "arena", srcs = [ @@ -264,6 +285,7 @@ cc_library( ":arena_allocation_policy", ":arena_cleanup", ":arena_config", + ":string_block", "//src/google/protobuf/stubs:lite", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", @@ -338,6 +360,7 @@ cc_library( ":arena", ":arena_align", ":arena_config", + ":string_block", "//src/google/protobuf/io", "//src/google/protobuf/stubs:lite", "@com_google_absl//absl/container:btree", diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 5b24882ccd..071360f14e 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -35,6 +35,7 @@ #include #include #include +#include #include #include "absl/base/attributes.h" @@ -163,6 +164,8 @@ void SerialArena::Init(ArenaBlock* b, size_t offset) { space_allocated_.store(b->size, std::memory_order_relaxed); cached_block_length_ = 0; cached_blocks_ = nullptr; + string_block_ = nullptr; + string_block_unused_.store(0, std::memory_order_relaxed); } SerialArena* SerialArena::New(SizedPtr mem, ThreadSafeArena& parent) { @@ -192,6 +195,25 @@ void* SerialArena::AllocateAlignedFallback(size_t n) { return AllocateFromExisting(n); } +PROTOBUF_NOINLINE +void* SerialArena::AllocateFromStringBlockFallback() { + if (string_block_) { + ABSL_DCHECK_EQ(string_block_unused_.load(std::memory_order_relaxed), 0U); + space_used_.store(space_used_.load(std::memory_order_relaxed) + + string_block_->effective_size(), + std::memory_order_relaxed); + } + + string_block_ = StringBlock::New(string_block_); + space_allocated_.store(space_allocated_.load(std::memory_order_relaxed) + + string_block_->allocated_size(), + std::memory_order_relaxed); + + size_t unused = string_block_->effective_size() - sizeof(std::string); + string_block_unused_.store(unused, std::memory_order_relaxed); + return string_block_->AtOffset(unused); +} + PROTOBUF_NOINLINE void* SerialArena::AllocateAlignedWithCleanupFallback( size_t n, size_t align, void (*destructor)(void*)) { @@ -256,17 +278,41 @@ uint64_t SerialArena::SpaceUsed() const { // usage of the *current* block. // TODO(mkruskal) Consider eliminating this race in exchange for a possible // performance hit on ARM (see cl/455186837). + uint64_t current_space_used = + string_block_ ? string_block_->effective_size() - + string_block_unused_.load(std::memory_order_relaxed) + : 0; const ArenaBlock* h = head_.load(std::memory_order_acquire); - if (h->IsSentry()) return 0; + if (h->IsSentry()) return current_space_used; const uint64_t current_block_size = h->size; - uint64_t current_space_used = std::min( + current_space_used += std::min( static_cast( ptr() - const_cast(h)->Pointer(kBlockHeaderSize)), current_block_size); return current_space_used + space_used_.load(std::memory_order_relaxed); } +size_t SerialArena::FreeStringBlocks(StringBlock* string_block, + size_t unused_bytes) { + ABSL_DCHECK(string_block != nullptr); + StringBlock* next = string_block->next(); + std::string* end = string_block->end(); + for (std::string* s = string_block->AtOffset(unused_bytes); s != end; ++s) { + s->~basic_string(); + } + size_t deallocated = StringBlock::Delete(string_block); + + while ((string_block = next) != nullptr) { + next = string_block->next(); + for (std::string& s : *string_block) { + s.~basic_string(); + } + deallocated += StringBlock::Delete(string_block); + } + return deallocated; +} + void SerialArena::CleanupList() { ArenaBlock* b = head(); if (b->IsSentry()) return; @@ -638,7 +684,7 @@ ThreadSafeArena::~ThreadSafeArena() { SizedPtr ThreadSafeArena::Free(size_t* space_allocated) { auto deallocator = GetDeallocator(alloc_policy_.get(), space_allocated); - WalkSerialArenaChunk([deallocator](SerialArenaChunk* chunk) { + WalkSerialArenaChunk([&](SerialArenaChunk* chunk) { absl::Span> span = chunk->arenas(); // Walks arenas backward to handle the first serial arena the last. Freeing // in reverse-order to the order in which objects were created may not be @@ -646,6 +692,8 @@ SizedPtr ThreadSafeArena::Free(size_t* space_allocated) { for (auto it = span.rbegin(); it != span.rend(); ++it) { SerialArena* serial = it->load(std::memory_order_relaxed); ABSL_DCHECK_NE(serial, nullptr); + // Free string blocks + *space_allocated += serial->FreeStringBlocks(); // Always frees the first block of "serial" as it cannot be user-provided. SizedPtr mem = serial->Free(deallocator); ABSL_DCHECK_NE(mem.p, nullptr); @@ -658,6 +706,7 @@ SizedPtr ThreadSafeArena::Free(size_t* space_allocated) { }); // The first block of the first arena is special and let the caller handle it. + *space_allocated += first_arena_.FreeStringBlocks(); return first_arena_.Free(deallocator); } @@ -717,6 +766,15 @@ void* ThreadSafeArena::AllocateAlignedWithCleanupFallback( ->AllocateAlignedWithCleanup(n, align, destructor); } +PROTOBUF_NOINLINE +void* ThreadSafeArena::AllocateFromStringBlock() { + SerialArena* arena; + if (PROTOBUF_PREDICT_FALSE(!GetSerialArenaFast(&arena))) { + arena = GetSerialArenaFallback(0); + } + return arena->AllocateFromStringBlock(); +} + template void ThreadSafeArena::WalkConstSerialArenaChunk(Functor fn) const { const SerialArenaChunk* chunk = head_.load(std::memory_order_acquire); diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 23e6999c4c..e9e7a15d4e 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -681,6 +681,11 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { friend struct internal::ArenaTestPeer; }; +template <> +inline void* Arena::AllocateInternal() { + return impl_.AllocateFromStringBlock(); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/serial_arena.h b/src/google/protobuf/serial_arena.h index e3f088ec6c..95767f6647 100644 --- a/src/google/protobuf/serial_arena.h +++ b/src/google/protobuf/serial_arena.h @@ -35,11 +35,13 @@ #include #include +#include #include #include #include #include "google/protobuf/stubs/common.h" +#include "absl/base/attributes.h" #include "absl/log/absl_check.h" #include "absl/numeric/bits.h" #include "google/protobuf/arena_align.h" @@ -47,6 +49,7 @@ #include "google/protobuf/arena_config.h" #include "google/protobuf/arenaz_sampler.h" #include "google/protobuf/port.h" +#include "google/protobuf/string_block.h" // Must be included last. @@ -106,6 +109,14 @@ struct FirstSerialArena { class PROTOBUF_EXPORT SerialArena { public: void CleanupList(); + size_t FreeStringBlocks() { + // On the active block delete all strings skipping the unused instances. + size_t unused_bytes = string_block_unused_.load(std::memory_order_relaxed); + if (string_block_ != nullptr) { + return FreeStringBlocks(string_block_, unused_bytes); + } + return 0; + } uint64_t SpaceAllocated() const { return space_allocated_.load(std::memory_order_relaxed); } @@ -280,7 +291,12 @@ class PROTOBUF_EXPORT SerialArena { AddCleanupFromExisting(elem, destructor); } + ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlock(); + private: + bool MaybeAllocateString(void*& p); + ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlockFallback(); + void* AllocateFromExistingWithCleanupFallback(size_t n, size_t align, void (*destructor)(void*)) { n = AlignUpTo(n, align); @@ -315,6 +331,8 @@ class PROTOBUF_EXPORT SerialArena { template SizedPtr Free(Deallocator deallocator); + static size_t FreeStringBlocks(StringBlock* string_block, size_t unused); + // Members are declared here to track sizeof(SerialArena) and hotness // centrally. They are (roughly) laid out in descending order of hotness. @@ -325,6 +343,14 @@ class PROTOBUF_EXPORT SerialArena { // Limiting address up to which memory can be allocated from the head block. char* limit_ = nullptr; + // The active string block. + StringBlock* string_block_ = nullptr; + + // The number of unused bytes in string_block_. + // We allocate from `effective_size()` down to 0 inside `string_block_`. + // `unused == 0` means that `string_block_` is exhausted. (or null). + std::atomic string_block_unused_{0}; + std::atomic head_{nullptr}; // Head of linked list of blocks. std::atomic space_used_{0}; // Necessary for metrics. std::atomic space_allocated_{0}; @@ -373,6 +399,32 @@ class PROTOBUF_EXPORT SerialArena { ArenaAlignDefault::Ceil(sizeof(ArenaBlock)); }; +inline PROTOBUF_ALWAYS_INLINE bool SerialArena::MaybeAllocateString(void*& p) { + // Check how many unused instances are in the current block. + size_t unused_bytes = string_block_unused_.load(std::memory_order_relaxed); + if (PROTOBUF_PREDICT_TRUE(unused_bytes != 0)) { + unused_bytes -= sizeof(std::string); + string_block_unused_.store(unused_bytes, std::memory_order_relaxed); + p = string_block_->AtOffset(unused_bytes); + return true; + } + return false; +} + +template <> +inline PROTOBUF_ALWAYS_INLINE void* +SerialArena::MaybeAllocateWithCleanup() { + void* p; + return MaybeAllocateString(p) ? p : nullptr; +} + +ABSL_ATTRIBUTE_RETURNS_NONNULL inline PROTOBUF_ALWAYS_INLINE void* +SerialArena::AllocateFromStringBlock() { + void* p; + if (ABSL_PREDICT_TRUE(MaybeAllocateString(p))) return p; + return AllocateFromStringBlockFallback(); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/string_block.h b/src/google/protobuf/string_block.h new file mode 100644 index 0000000000..fd7812b9fe --- /dev/null +++ b/src/google/protobuf/string_block.h @@ -0,0 +1,164 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// This file defines the internal StringBlock class + +#ifndef GOOGLE_PROTOBUF_STRING_BLOCK_H__ +#define GOOGLE_PROTOBUF_STRING_BLOCK_H__ + +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/log/absl_check.h" +#include "google/protobuf/arena_align.h" +#include "google/protobuf/port.h" + +// Must be included last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace internal { + +// StringBlock provides heap allocated, dynamically sized blocks (mini arenas) +// for allocating std::string instances. StringBlocks are allocated through +// the `New` function, and must be freed using the `Delete` function. +// StringBlocks are automatically sized from 256B to 8KB depending on the +// `next` instance provided in the `New` function to keep the average maximum +// unused space limited to 25%, or up to 4KB. +class alignas(std::string) StringBlock { + public: + StringBlock() = delete; + StringBlock(const StringBlock&) = delete; + StringBlock& operator=(const StringBlock&) = delete; + + // Allocates a new StringBlock pointing to `next`, which can be null. + // The size of the returned block depends on the allocated size of `next`. + static StringBlock* New(StringBlock* next); + + // Deletes `block`. `block` must not be null. + static size_t Delete(StringBlock* block); + + StringBlock* next() const; + + // Returns the string instance at offset `offset`. + // `offset` must be a multiple of sizeof(std::string), and be less than or + // equal to `effective_size()`. `AtOffset(effective_size())` returns the + // end of the allocated string instances and must not be de-referenced. + ABSL_ATTRIBUTE_RETURNS_NONNULL std::string* AtOffset(size_t offset); + + // Returns a pointer to the first string instance in this block. + ABSL_ATTRIBUTE_RETURNS_NONNULL std::string* begin(); + + // Returns a pointer directly beyond the last string instance in this block. + ABSL_ATTRIBUTE_RETURNS_NONNULL std::string* end(); + + // Returns the total allocation size of this instance. + size_t allocated_size() const { return allocated_size_; } + + // Returns the effective size available for allocation string instances. + // This value is guaranteed to be a multiple of sizeof(std::string), and + // guaranteed to never be zero. + size_t effective_size() const; + + private: + static_assert(alignof(std::string) <= sizeof(void*), ""); + static_assert(alignof(std::string) <= ArenaAlignDefault::align, ""); + + ~StringBlock() = default; + + explicit StringBlock(StringBlock* next, uint32_t size, + uint32_t next_size) noexcept + : next_(next), allocated_size_(size), next_size_(next_size) {} + + static constexpr uint32_t min_size() { return size_t{256}; } + static constexpr uint32_t max_size() { return size_t{8192}; } + + // Returns the size of the next block. + size_t next_size() const { return next_size_; } + + StringBlock* const next_; + const uint32_t allocated_size_; + const uint32_t next_size_; +}; + +inline StringBlock* StringBlock::New(StringBlock* next) { + // Compute required size, rounding down to a multiple of sizeof(std:string) + // so that we can optimize the allocation path. I.e., we incur a (constant + // size) MOD() operation cost here to avoid any MUL() later on. + uint32_t size = min_size(); + uint32_t next_size = min_size(); + if (next) { + size = next->next_size_; + next_size = std::min(size * 2, max_size()); + } + size -= (size - sizeof(StringBlock)) % sizeof(std::string); + void* p = ::operator new(size); + return new (p) StringBlock(next, size, next_size); +} + +inline size_t StringBlock::Delete(StringBlock* block) { + ABSL_DCHECK(block != nullptr); + size_t size = block->allocated_size(); + internal::SizedDelete(block, size); + return size; +} + +inline StringBlock* StringBlock::next() const { return next_; } + +inline size_t StringBlock::effective_size() const { + return allocated_size_ - sizeof(StringBlock); +} + +ABSL_ATTRIBUTE_RETURNS_NONNULL inline std::string* StringBlock::AtOffset( + size_t offset) { + ABSL_DCHECK_LE(offset, effective_size()); + return reinterpret_cast(reinterpret_cast(this + 1) + + offset); +} + +ABSL_ATTRIBUTE_RETURNS_NONNULL inline std::string* StringBlock::begin() { + return AtOffset(0); +} + +ABSL_ATTRIBUTE_RETURNS_NONNULL inline std::string* StringBlock::end() { + return AtOffset(effective_size()); +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" + +#endif // GOOGLE_PROTOBUF_STRING_BLOCK_H__ diff --git a/src/google/protobuf/string_block_test.cc b/src/google/protobuf/string_block_test.cc new file mode 100644 index 0000000000..ff1c71fd89 --- /dev/null +++ b/src/google/protobuf/string_block_test.cc @@ -0,0 +1,108 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// This file defines tests for the internal StringBlock class + +#include "google/protobuf/string_block.h" + +#include +#include + +#include +#include + +// Must be included last +#include "google/protobuf/port_def.inc" + +using ::testing::Eq; +using ::testing::Ne; + +namespace google { +namespace protobuf { +namespace internal { +namespace { + +size_t EffectiveSizeFor(int size) { + size -= sizeof(StringBlock); + return static_cast(size - (size % sizeof(std::string))); +} + +size_t AllocatedSizeFor(int size) { + return EffectiveSizeFor(size) + sizeof(StringBlock); +} + +TEST(StringBlockTest, OneNewBlock) { + StringBlock* block = StringBlock::New(nullptr); + ASSERT_THAT(block, Ne(nullptr)); + EXPECT_THAT(block->next(), Eq(nullptr)); + EXPECT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(256))); + EXPECT_THAT(block->effective_size(), Eq(EffectiveSizeFor(256))); + EXPECT_THAT(block->begin(), Eq(block->AtOffset(0))); + EXPECT_THAT(block->end(), Eq(block->AtOffset(block->effective_size()))); + + EXPECT_THAT(StringBlock::Delete(block), Eq(AllocatedSizeFor(256))); +} + +TEST(StringBlockTest, NewBlocks) { + // Note: first two blocks are 256 + StringBlock* previous = StringBlock::New(nullptr); + + for (int size = 256; size <= 8192; size *= 2) { + StringBlock* block = StringBlock::New(previous); + ASSERT_THAT(block, Ne(nullptr)); + ASSERT_THAT(block->next(), Eq(previous)); + ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(size))); + ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(size))); + ASSERT_THAT(block->begin(), Eq(block->AtOffset(0))); + ASSERT_THAT(block->end(), Eq(block->AtOffset(block->effective_size()))); + previous = block; + } + + // Capped at 8K from here on + StringBlock* block = StringBlock::New(previous); + ASSERT_THAT(block, Ne(nullptr)); + EXPECT_THAT(block->next(), Eq(previous)); + ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(8192))); + ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(8192))); + ASSERT_THAT(block->begin(), Eq(block->AtOffset(0))); + ASSERT_THAT(block->end(), Eq(block->AtOffset(block->effective_size()))); + + while (block) { + size_t size = block->allocated_size(); + StringBlock* next = block->next(); + EXPECT_THAT(StringBlock::Delete(block), Eq(AllocatedSizeFor(size))); + block = next; + } +} + +} // namespace +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/thread_safe_arena.h b/src/google/protobuf/thread_safe_arena.h index fa2be1b1cf..d2a3e6e72a 100644 --- a/src/google/protobuf/thread_safe_arena.h +++ b/src/google/protobuf/thread_safe_arena.h @@ -124,6 +124,8 @@ class PROTOBUF_EXPORT ThreadSafeArena { // Add object pointer and cleanup function pointer to the list. void AddCleanup(void* elem, void (*cleanup)(void*)); + void* AllocateFromStringBlock(); + private: friend class ArenaBenchmark; friend class TcParser;