Add StringBlock logic to arenas

This change introduces a dedicated StringBlock class that hosts block allocated string instances which are destroyed when the arena gets destroyed.

Experiments have shown this to save 0.40% in memory and 0.10% of CPU in arena heavy use cases.

PiperOrigin-RevId: 508205452
pull/11867/head
Martijn Vels 2 years ago committed by Copybara-Service
parent 4c681aad2f
commit c8e7ac7b5d
  1. 23
      src/google/protobuf/BUILD.bazel
  2. 64
      src/google/protobuf/arena.cc
  3. 5
      src/google/protobuf/arena.h
  4. 52
      src/google/protobuf/serial_arena.h
  5. 164
      src/google/protobuf/string_block.h
  6. 108
      src/google/protobuf/string_block_test.cc
  7. 2
      src/google/protobuf/thread_safe_arena.h

@ -242,6 +242,27 @@ cc_library(
],
)
cc_library(
name = "string_block",
hdrs = ["string_block.h"],
include_prefix = "google/protobuf",
deps = [
":arena_align",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/log:absl_check",
],
)
cc_test(
name = "string_block_test",
srcs = ["string_block_test.cc"],
deps = [
":string_block",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "arena",
srcs = [
@ -264,6 +285,7 @@ cc_library(
":arena_allocation_policy",
":arena_cleanup",
":arena_config",
":string_block",
"//src/google/protobuf/stubs:lite",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
@ -338,6 +360,7 @@ cc_library(
":arena",
":arena_align",
":arena_config",
":string_block",
"//src/google/protobuf/io",
"//src/google/protobuf/stubs:lite",
"@com_google_absl//absl/container:btree",

@ -35,6 +35,7 @@
#include <cstddef>
#include <cstdint>
#include <limits>
#include <string>
#include <typeinfo>
#include "absl/base/attributes.h"
@ -163,6 +164,8 @@ void SerialArena::Init(ArenaBlock* b, size_t offset) {
space_allocated_.store(b->size, std::memory_order_relaxed);
cached_block_length_ = 0;
cached_blocks_ = nullptr;
string_block_ = nullptr;
string_block_unused_.store(0, std::memory_order_relaxed);
}
SerialArena* SerialArena::New(SizedPtr mem, ThreadSafeArena& parent) {
@ -192,6 +195,25 @@ void* SerialArena::AllocateAlignedFallback(size_t n) {
return AllocateFromExisting(n);
}
PROTOBUF_NOINLINE
void* SerialArena::AllocateFromStringBlockFallback() {
if (string_block_) {
ABSL_DCHECK_EQ(string_block_unused_.load(std::memory_order_relaxed), 0U);
space_used_.store(space_used_.load(std::memory_order_relaxed) +
string_block_->effective_size(),
std::memory_order_relaxed);
}
string_block_ = StringBlock::New(string_block_);
space_allocated_.store(space_allocated_.load(std::memory_order_relaxed) +
string_block_->allocated_size(),
std::memory_order_relaxed);
size_t unused = string_block_->effective_size() - sizeof(std::string);
string_block_unused_.store(unused, std::memory_order_relaxed);
return string_block_->AtOffset(unused);
}
PROTOBUF_NOINLINE
void* SerialArena::AllocateAlignedWithCleanupFallback(
size_t n, size_t align, void (*destructor)(void*)) {
@ -256,17 +278,41 @@ uint64_t SerialArena::SpaceUsed() const {
// usage of the *current* block.
// TODO(mkruskal) Consider eliminating this race in exchange for a possible
// performance hit on ARM (see cl/455186837).
uint64_t current_space_used =
string_block_ ? string_block_->effective_size() -
string_block_unused_.load(std::memory_order_relaxed)
: 0;
const ArenaBlock* h = head_.load(std::memory_order_acquire);
if (h->IsSentry()) return 0;
if (h->IsSentry()) return current_space_used;
const uint64_t current_block_size = h->size;
uint64_t current_space_used = std::min(
current_space_used += std::min(
static_cast<uint64_t>(
ptr() - const_cast<ArenaBlock*>(h)->Pointer(kBlockHeaderSize)),
current_block_size);
return current_space_used + space_used_.load(std::memory_order_relaxed);
}
size_t SerialArena::FreeStringBlocks(StringBlock* string_block,
size_t unused_bytes) {
ABSL_DCHECK(string_block != nullptr);
StringBlock* next = string_block->next();
std::string* end = string_block->end();
for (std::string* s = string_block->AtOffset(unused_bytes); s != end; ++s) {
s->~basic_string();
}
size_t deallocated = StringBlock::Delete(string_block);
while ((string_block = next) != nullptr) {
next = string_block->next();
for (std::string& s : *string_block) {
s.~basic_string();
}
deallocated += StringBlock::Delete(string_block);
}
return deallocated;
}
void SerialArena::CleanupList() {
ArenaBlock* b = head();
if (b->IsSentry()) return;
@ -638,7 +684,7 @@ ThreadSafeArena::~ThreadSafeArena() {
SizedPtr ThreadSafeArena::Free(size_t* space_allocated) {
auto deallocator = GetDeallocator(alloc_policy_.get(), space_allocated);
WalkSerialArenaChunk([deallocator](SerialArenaChunk* chunk) {
WalkSerialArenaChunk([&](SerialArenaChunk* chunk) {
absl::Span<std::atomic<SerialArena*>> span = chunk->arenas();
// Walks arenas backward to handle the first serial arena the last. Freeing
// in reverse-order to the order in which objects were created may not be
@ -646,6 +692,8 @@ SizedPtr ThreadSafeArena::Free(size_t* space_allocated) {
for (auto it = span.rbegin(); it != span.rend(); ++it) {
SerialArena* serial = it->load(std::memory_order_relaxed);
ABSL_DCHECK_NE(serial, nullptr);
// Free string blocks
*space_allocated += serial->FreeStringBlocks();
// Always frees the first block of "serial" as it cannot be user-provided.
SizedPtr mem = serial->Free(deallocator);
ABSL_DCHECK_NE(mem.p, nullptr);
@ -658,6 +706,7 @@ SizedPtr ThreadSafeArena::Free(size_t* space_allocated) {
});
// The first block of the first arena is special and let the caller handle it.
*space_allocated += first_arena_.FreeStringBlocks();
return first_arena_.Free(deallocator);
}
@ -717,6 +766,15 @@ void* ThreadSafeArena::AllocateAlignedWithCleanupFallback(
->AllocateAlignedWithCleanup(n, align, destructor);
}
PROTOBUF_NOINLINE
void* ThreadSafeArena::AllocateFromStringBlock() {
SerialArena* arena;
if (PROTOBUF_PREDICT_FALSE(!GetSerialArenaFast(&arena))) {
arena = GetSerialArenaFallback(0);
}
return arena->AllocateFromStringBlock();
}
template <typename Functor>
void ThreadSafeArena::WalkConstSerialArenaChunk(Functor fn) const {
const SerialArenaChunk* chunk = head_.load(std::memory_order_acquire);

@ -681,6 +681,11 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
friend struct internal::ArenaTestPeer;
};
template <>
inline void* Arena::AllocateInternal<std::string, false>() {
return impl_.AllocateFromStringBlock();
}
} // namespace protobuf
} // namespace google

@ -35,11 +35,13 @@
#include <algorithm>
#include <atomic>
#include <string>
#include <type_traits>
#include <typeinfo>
#include <utility>
#include "google/protobuf/stubs/common.h"
#include "absl/base/attributes.h"
#include "absl/log/absl_check.h"
#include "absl/numeric/bits.h"
#include "google/protobuf/arena_align.h"
@ -47,6 +49,7 @@
#include "google/protobuf/arena_config.h"
#include "google/protobuf/arenaz_sampler.h"
#include "google/protobuf/port.h"
#include "google/protobuf/string_block.h"
// Must be included last.
@ -106,6 +109,14 @@ struct FirstSerialArena {
class PROTOBUF_EXPORT SerialArena {
public:
void CleanupList();
size_t FreeStringBlocks() {
// On the active block delete all strings skipping the unused instances.
size_t unused_bytes = string_block_unused_.load(std::memory_order_relaxed);
if (string_block_ != nullptr) {
return FreeStringBlocks(string_block_, unused_bytes);
}
return 0;
}
uint64_t SpaceAllocated() const {
return space_allocated_.load(std::memory_order_relaxed);
}
@ -280,7 +291,12 @@ class PROTOBUF_EXPORT SerialArena {
AddCleanupFromExisting(elem, destructor);
}
ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlock();
private:
bool MaybeAllocateString(void*& p);
ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlockFallback();
void* AllocateFromExistingWithCleanupFallback(size_t n, size_t align,
void (*destructor)(void*)) {
n = AlignUpTo(n, align);
@ -315,6 +331,8 @@ class PROTOBUF_EXPORT SerialArena {
template <typename Deallocator>
SizedPtr Free(Deallocator deallocator);
static size_t FreeStringBlocks(StringBlock* string_block, size_t unused);
// Members are declared here to track sizeof(SerialArena) and hotness
// centrally. They are (roughly) laid out in descending order of hotness.
@ -325,6 +343,14 @@ class PROTOBUF_EXPORT SerialArena {
// Limiting address up to which memory can be allocated from the head block.
char* limit_ = nullptr;
// The active string block.
StringBlock* string_block_ = nullptr;
// The number of unused bytes in string_block_.
// We allocate from `effective_size()` down to 0 inside `string_block_`.
// `unused == 0` means that `string_block_` is exhausted. (or null).
std::atomic<size_t> string_block_unused_{0};
std::atomic<ArenaBlock*> head_{nullptr}; // Head of linked list of blocks.
std::atomic<size_t> space_used_{0}; // Necessary for metrics.
std::atomic<size_t> space_allocated_{0};
@ -373,6 +399,32 @@ class PROTOBUF_EXPORT SerialArena {
ArenaAlignDefault::Ceil(sizeof(ArenaBlock));
};
inline PROTOBUF_ALWAYS_INLINE bool SerialArena::MaybeAllocateString(void*& p) {
// Check how many unused instances are in the current block.
size_t unused_bytes = string_block_unused_.load(std::memory_order_relaxed);
if (PROTOBUF_PREDICT_TRUE(unused_bytes != 0)) {
unused_bytes -= sizeof(std::string);
string_block_unused_.store(unused_bytes, std::memory_order_relaxed);
p = string_block_->AtOffset(unused_bytes);
return true;
}
return false;
}
template <>
inline PROTOBUF_ALWAYS_INLINE void*
SerialArena::MaybeAllocateWithCleanup<std::string>() {
void* p;
return MaybeAllocateString(p) ? p : nullptr;
}
ABSL_ATTRIBUTE_RETURNS_NONNULL inline PROTOBUF_ALWAYS_INLINE void*
SerialArena::AllocateFromStringBlock() {
void* p;
if (ABSL_PREDICT_TRUE(MaybeAllocateString(p))) return p;
return AllocateFromStringBlockFallback();
}
} // namespace internal
} // namespace protobuf
} // namespace google

@ -0,0 +1,164 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// This file defines the internal StringBlock class
#ifndef GOOGLE_PROTOBUF_STRING_BLOCK_H__
#define GOOGLE_PROTOBUF_STRING_BLOCK_H__
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <string>
#include "absl/base/attributes.h"
#include "absl/log/absl_check.h"
#include "google/protobuf/arena_align.h"
#include "google/protobuf/port.h"
// Must be included last.
#include "google/protobuf/port_def.inc"
namespace google {
namespace protobuf {
namespace internal {
// StringBlock provides heap allocated, dynamically sized blocks (mini arenas)
// for allocating std::string instances. StringBlocks are allocated through
// the `New` function, and must be freed using the `Delete` function.
// StringBlocks are automatically sized from 256B to 8KB depending on the
// `next` instance provided in the `New` function to keep the average maximum
// unused space limited to 25%, or up to 4KB.
class alignas(std::string) StringBlock {
public:
StringBlock() = delete;
StringBlock(const StringBlock&) = delete;
StringBlock& operator=(const StringBlock&) = delete;
// Allocates a new StringBlock pointing to `next`, which can be null.
// The size of the returned block depends on the allocated size of `next`.
static StringBlock* New(StringBlock* next);
// Deletes `block`. `block` must not be null.
static size_t Delete(StringBlock* block);
StringBlock* next() const;
// Returns the string instance at offset `offset`.
// `offset` must be a multiple of sizeof(std::string), and be less than or
// equal to `effective_size()`. `AtOffset(effective_size())` returns the
// end of the allocated string instances and must not be de-referenced.
ABSL_ATTRIBUTE_RETURNS_NONNULL std::string* AtOffset(size_t offset);
// Returns a pointer to the first string instance in this block.
ABSL_ATTRIBUTE_RETURNS_NONNULL std::string* begin();
// Returns a pointer directly beyond the last string instance in this block.
ABSL_ATTRIBUTE_RETURNS_NONNULL std::string* end();
// Returns the total allocation size of this instance.
size_t allocated_size() const { return allocated_size_; }
// Returns the effective size available for allocation string instances.
// This value is guaranteed to be a multiple of sizeof(std::string), and
// guaranteed to never be zero.
size_t effective_size() const;
private:
static_assert(alignof(std::string) <= sizeof(void*), "");
static_assert(alignof(std::string) <= ArenaAlignDefault::align, "");
~StringBlock() = default;
explicit StringBlock(StringBlock* next, uint32_t size,
uint32_t next_size) noexcept
: next_(next), allocated_size_(size), next_size_(next_size) {}
static constexpr uint32_t min_size() { return size_t{256}; }
static constexpr uint32_t max_size() { return size_t{8192}; }
// Returns the size of the next block.
size_t next_size() const { return next_size_; }
StringBlock* const next_;
const uint32_t allocated_size_;
const uint32_t next_size_;
};
inline StringBlock* StringBlock::New(StringBlock* next) {
// Compute required size, rounding down to a multiple of sizeof(std:string)
// so that we can optimize the allocation path. I.e., we incur a (constant
// size) MOD() operation cost here to avoid any MUL() later on.
uint32_t size = min_size();
uint32_t next_size = min_size();
if (next) {
size = next->next_size_;
next_size = std::min(size * 2, max_size());
}
size -= (size - sizeof(StringBlock)) % sizeof(std::string);
void* p = ::operator new(size);
return new (p) StringBlock(next, size, next_size);
}
inline size_t StringBlock::Delete(StringBlock* block) {
ABSL_DCHECK(block != nullptr);
size_t size = block->allocated_size();
internal::SizedDelete(block, size);
return size;
}
inline StringBlock* StringBlock::next() const { return next_; }
inline size_t StringBlock::effective_size() const {
return allocated_size_ - sizeof(StringBlock);
}
ABSL_ATTRIBUTE_RETURNS_NONNULL inline std::string* StringBlock::AtOffset(
size_t offset) {
ABSL_DCHECK_LE(offset, effective_size());
return reinterpret_cast<std::string*>(reinterpret_cast<char*>(this + 1) +
offset);
}
ABSL_ATTRIBUTE_RETURNS_NONNULL inline std::string* StringBlock::begin() {
return AtOffset(0);
}
ABSL_ATTRIBUTE_RETURNS_NONNULL inline std::string* StringBlock::end() {
return AtOffset(effective_size());
}
} // namespace internal
} // namespace protobuf
} // namespace google
#include "google/protobuf/port_undef.inc"
#endif // GOOGLE_PROTOBUF_STRING_BLOCK_H__

@ -0,0 +1,108 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// This file defines tests for the internal StringBlock class
#include "google/protobuf/string_block.h"
#include <cstddef>
#include <string>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
// Must be included last
#include "google/protobuf/port_def.inc"
using ::testing::Eq;
using ::testing::Ne;
namespace google {
namespace protobuf {
namespace internal {
namespace {
size_t EffectiveSizeFor(int size) {
size -= sizeof(StringBlock);
return static_cast<size_t>(size - (size % sizeof(std::string)));
}
size_t AllocatedSizeFor(int size) {
return EffectiveSizeFor(size) + sizeof(StringBlock);
}
TEST(StringBlockTest, OneNewBlock) {
StringBlock* block = StringBlock::New(nullptr);
ASSERT_THAT(block, Ne(nullptr));
EXPECT_THAT(block->next(), Eq(nullptr));
EXPECT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(256)));
EXPECT_THAT(block->effective_size(), Eq(EffectiveSizeFor(256)));
EXPECT_THAT(block->begin(), Eq(block->AtOffset(0)));
EXPECT_THAT(block->end(), Eq(block->AtOffset(block->effective_size())));
EXPECT_THAT(StringBlock::Delete(block), Eq(AllocatedSizeFor(256)));
}
TEST(StringBlockTest, NewBlocks) {
// Note: first two blocks are 256
StringBlock* previous = StringBlock::New(nullptr);
for (int size = 256; size <= 8192; size *= 2) {
StringBlock* block = StringBlock::New(previous);
ASSERT_THAT(block, Ne(nullptr));
ASSERT_THAT(block->next(), Eq(previous));
ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(size)));
ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(size)));
ASSERT_THAT(block->begin(), Eq(block->AtOffset(0)));
ASSERT_THAT(block->end(), Eq(block->AtOffset(block->effective_size())));
previous = block;
}
// Capped at 8K from here on
StringBlock* block = StringBlock::New(previous);
ASSERT_THAT(block, Ne(nullptr));
EXPECT_THAT(block->next(), Eq(previous));
ASSERT_THAT(block->allocated_size(), Eq(AllocatedSizeFor(8192)));
ASSERT_THAT(block->effective_size(), Eq(EffectiveSizeFor(8192)));
ASSERT_THAT(block->begin(), Eq(block->AtOffset(0)));
ASSERT_THAT(block->end(), Eq(block->AtOffset(block->effective_size())));
while (block) {
size_t size = block->allocated_size();
StringBlock* next = block->next();
EXPECT_THAT(StringBlock::Delete(block), Eq(AllocatedSizeFor(size)));
block = next;
}
}
} // namespace
} // namespace internal
} // namespace protobuf
} // namespace google

@ -124,6 +124,8 @@ class PROTOBUF_EXPORT ThreadSafeArena {
// Add object pointer and cleanup function pointer to the list.
void AddCleanup(void* elem, void (*cleanup)(void*));
void* AllocateFromStringBlock();
private:
friend class ArenaBenchmark;
friend class TcParser;

Loading…
Cancel
Save