Abseil Common Libraries (C++) (grcp 依赖) https://abseil.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

543 lines
19 KiB

// Copyright 2021 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_
#define ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_
#include <atomic>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <type_traits>
#include "absl/base/config.h"
#include "absl/base/internal/endian.h"
#include "absl/base/internal/invoke.h"
#include "absl/base/optimization.h"
#include "absl/container/internal/compressed_tuple.h"
#include "absl/meta/type_traits.h"
#include "absl/strings/string_view.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace cord_internal {
class CordzInfo;
// Default feature enable states for cord ring buffers
enum CordFeatureDefaults {
kCordEnableRingBufferDefault = false,
kCordShallowSubcordsDefault = false
};
extern std::atomic<bool> cord_ring_buffer_enabled;
extern std::atomic<bool> shallow_subcords_enabled;
inline void enable_cord_ring_buffer(bool enable) {
cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed);
}
inline void enable_shallow_subcords(bool enable) {
shallow_subcords_enabled.store(enable, std::memory_order_relaxed);
}
enum Constants {
// The inlined size to use with absl::InlinedVector.
//
// Note: The InlinedVectors in this file (and in cord.h) do not need to use
// the same value for their inlined size. The fact that they do is historical.
// It may be desirable for each to use a different inlined size optimized for
// that InlinedVector's usage.
//
// TODO(jgm): Benchmark to see if there's a more optimal value than 47 for
// the inlined vector size (47 exists for backward compatibility).
kInlinedVectorSize = 47,
// Prefer copying blocks of at most this size, otherwise reference count.
kMaxBytesToCopy = 511
};
// Wraps std::atomic for reference counting.
class Refcount {
public:
constexpr Refcount() : count_{kRefIncrement} {}
struct Immortal {};
explicit constexpr Refcount(Immortal) : count_(kImmortalTag) {}
// Increments the reference count. Imposes no memory ordering.
inline void Increment() {
count_.fetch_add(kRefIncrement, std::memory_order_relaxed);
}
// Asserts that the current refcount is greater than 0. If the refcount is
// greater than 1, decrements the reference count.
//
// Returns false if there are no references outstanding; true otherwise.
// Inserts barriers to ensure that state written before this method returns
// false will be visible to a thread that just observed this method returning
// false.
inline bool Decrement() {
int32_t refcount = count_.load(std::memory_order_acquire);
assert(refcount > 0 || refcount & kImmortalTag);
return refcount != kRefIncrement &&
count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) !=
kRefIncrement;
}
// Same as Decrement but expect that refcount is greater than 1.
inline bool DecrementExpectHighRefcount() {
int32_t refcount =
count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel);
assert(refcount > 0 || refcount & kImmortalTag);
return refcount != kRefIncrement;
}
// Returns the current reference count using acquire semantics.
inline int32_t Get() const {
return count_.load(std::memory_order_acquire) >> kImmortalShift;
}
// Returns whether the atomic integer is 1.
// If the reference count is used in the conventional way, a
// reference count of 1 implies that the current thread owns the
// reference and no other thread shares it.
// This call performs the test for a reference count of one, and
// performs the memory barrier needed for the owning thread
// to act on the object, knowing that it has exclusive access to the
// object.
inline bool IsOne() {
return count_.load(std::memory_order_acquire) == kRefIncrement;
}
bool IsImmortal() const {
return (count_.load(std::memory_order_relaxed) & kImmortalTag) != 0;
}
private:
// We reserve the bottom bit to tag a reference count as immortal.
// By making it `1` we ensure that we never reach `0` when adding/subtracting
// `2`, thus it never looks as if it should be destroyed.
// These are used for the StringConstant constructor where we do not increase
// the refcount at construction time (due to constinit requirements) but we
// will still decrease it at destruction time to avoid branching on Unref.
enum {
kImmortalShift = 1,
kRefIncrement = 1 << kImmortalShift,
kImmortalTag = kRefIncrement - 1
};
std::atomic<int32_t> count_;
};
// The overhead of a vtable is too much for Cord, so we roll our own subclasses
// using only a single byte to differentiate classes from each other - the "tag"
// byte. Define the subclasses first so we can provide downcasting helper
// functions in the base class.
struct CordRepConcat;
struct CordRepExternal;
struct CordRepFlat;
struct CordRepSubstring;
class CordRepRing;
// Various representations that we allow
enum CordRepKind {
CONCAT = 0,
EXTERNAL = 1,
SUBSTRING = 2,
RING = 3,
// We have different tags for different sized flat arrays,
// starting with FLAT, and limited to MAX_FLAT_TAG. The 224 value is based on
// the current 'size to tag' encoding of 8 / 32 bytes. If a new tag is needed
// in the future, then 'FLAT' and 'MAX_FLAT_TAG' should be adjusted as well
// as the Tag <---> Size logic so that FLAT stil represents the minimum flat
// allocation size. (32 bytes as of now).
FLAT = 4,
MAX_FLAT_TAG = 224
};
struct CordRep {
CordRep() = default;
constexpr CordRep(Refcount::Immortal immortal, size_t l)
: length(l), refcount(immortal), tag(EXTERNAL), storage{} {}
// The following three fields have to be less than 32 bytes since
// that is the smallest supported flat node size.
size_t length;
Refcount refcount;
// If tag < FLAT, it represents CordRepKind and indicates the type of node.
// Otherwise, the node type is CordRepFlat and the tag is the encoded size.
uint8_t tag;
char storage[1]; // Starting point for flat array: MUST BE LAST FIELD
inline CordRepRing* ring();
inline const CordRepRing* ring() const;
inline CordRepConcat* concat();
inline const CordRepConcat* concat() const;
inline CordRepSubstring* substring();
inline const CordRepSubstring* substring() const;
inline CordRepExternal* external();
inline const CordRepExternal* external() const;
inline CordRepFlat* flat();
inline const CordRepFlat* flat() const;
// --------------------------------------------------------------------
// Memory management
// Destroys the provided `rep`.
static void Destroy(CordRep* rep);
// Increments the reference count of `rep`.
// Requires `rep` to be a non-null pointer value.
static inline CordRep* Ref(CordRep* rep);
// Decrements the reference count of `rep`. Destroys rep if count reaches
// zero. Requires `rep` to be a non-null pointer value.
static inline void Unref(CordRep* rep);
};
struct CordRepConcat : public CordRep {
CordRep* left;
CordRep* right;
uint8_t depth() const { return static_cast<uint8_t>(storage[0]); }
void set_depth(uint8_t depth) { storage[0] = static_cast<char>(depth); }
};
struct CordRepSubstring : public CordRep {
size_t start; // Starting offset of substring in child
CordRep* child;
};
// Type for function pointer that will invoke the releaser function and also
// delete the `CordRepExternalImpl` corresponding to the passed in
// `CordRepExternal`.
using ExternalReleaserInvoker = void (*)(CordRepExternal*);
// External CordReps are allocated together with a type erased releaser. The
// releaser is stored in the memory directly following the CordRepExternal.
struct CordRepExternal : public CordRep {
CordRepExternal() = default;
explicit constexpr CordRepExternal(absl::string_view str)
: CordRep(Refcount::Immortal{}, str.size()),
base(str.data()),
releaser_invoker(nullptr) {}
const char* base;
// Pointer to function that knows how to call and destroy the releaser.
ExternalReleaserInvoker releaser_invoker;
// Deletes (releases) the external rep.
// Requires rep != nullptr and rep->tag == EXTERNAL
static void Delete(CordRep* rep);
};
struct Rank1 {};
struct Rank0 : Rank1 {};
template <typename Releaser, typename = ::absl::base_internal::invoke_result_t<
Releaser, absl::string_view>>
void InvokeReleaser(Rank0, Releaser&& releaser, absl::string_view data) {
::absl::base_internal::invoke(std::forward<Releaser>(releaser), data);
}
template <typename Releaser,
typename = ::absl::base_internal::invoke_result_t<Releaser>>
void InvokeReleaser(Rank1, Releaser&& releaser, absl::string_view) {
::absl::base_internal::invoke(std::forward<Releaser>(releaser));
}
// We use CompressedTuple so that we can benefit from EBCO.
template <typename Releaser>
struct CordRepExternalImpl
: public CordRepExternal,
public ::absl::container_internal::CompressedTuple<Releaser> {
// The extra int arg is so that we can avoid interfering with copy/move
// constructors while still benefitting from perfect forwarding.
template <typename T>
CordRepExternalImpl(T&& releaser, int)
: CordRepExternalImpl::CompressedTuple(std::forward<T>(releaser)) {
this->releaser_invoker = &Release;
}
~CordRepExternalImpl() {
InvokeReleaser(Rank0{}, std::move(this->template get<0>()),
absl::string_view(base, length));
}
static void Release(CordRepExternal* rep) {
delete static_cast<CordRepExternalImpl*>(rep);
}
};
inline void CordRepExternal::Delete(CordRep* rep) {
assert(rep != nullptr && rep->tag == EXTERNAL);
auto* rep_external = static_cast<CordRepExternal*>(rep);
assert(rep_external->releaser_invoker != nullptr);
rep_external->releaser_invoker(rep_external);
}
template <typename Str>
struct ConstInitExternalStorage {
ABSL_CONST_INIT static CordRepExternal value;
};
template <typename Str>
CordRepExternal ConstInitExternalStorage<Str>::value(Str::value);
enum {
kMaxInline = 15,
};
constexpr char GetOrNull(absl::string_view data, size_t pos) {
return pos < data.size() ? data[pos] : '\0';
}
// We store cordz_info as 64 bit pointer value in big endian format. This
// guarantees that the least significant byte of cordz_info matches the last
// byte of the inline data representation in as_chars_, which holds the inlined
// size or the 'is_tree' bit.
using cordz_info_t = int64_t;
// Assert that the `cordz_info` pointer value perfectly overlaps the last half
// of `as_chars_` and can hold a pointer value.
static_assert(sizeof(cordz_info_t) * 2 == kMaxInline + 1, "");
static_assert(sizeof(cordz_info_t) >= sizeof(intptr_t), "");
// BigEndianByte() creates a big endian representation of 'value', i.e.: a big
// endian value where the last byte in the host's representation holds 'value`,
// with all other bytes being 0.
static constexpr cordz_info_t BigEndianByte(unsigned char value) {
#if defined(ABSL_IS_BIG_ENDIAN)
return value;
#else
return static_cast<cordz_info_t>(value) << ((sizeof(cordz_info_t) - 1) * 8);
#endif
}
class InlineData {
public:
// kNullCordzInfo holds the big endian representation of intptr_t(1)
// This is the 'null' / initial value of 'cordz_info'. The null value
// is specifically big endian 1 as with 64-bit pointers, the last
// byte of cordz_info overlaps with the last byte holding the tag.
static constexpr cordz_info_t kNullCordzInfo = BigEndianByte(1);
// kFakeCordzInfo holds a 'fake', non-null cordz-info value we use to
// emulate the previous 'kProfiled' tag logic in 'set_profiled' until
// cord code is changed to store cordz_info values in InlineData.
static constexpr cordz_info_t kFakeCordzInfo = BigEndianByte(9);
constexpr InlineData() : as_chars_{0} {}
explicit constexpr InlineData(CordRep* rep) : as_tree_(rep) {}
explicit constexpr InlineData(absl::string_view chars)
: as_chars_{
GetOrNull(chars, 0), GetOrNull(chars, 1),
GetOrNull(chars, 2), GetOrNull(chars, 3),
GetOrNull(chars, 4), GetOrNull(chars, 5),
GetOrNull(chars, 6), GetOrNull(chars, 7),
GetOrNull(chars, 8), GetOrNull(chars, 9),
GetOrNull(chars, 10), GetOrNull(chars, 11),
GetOrNull(chars, 12), GetOrNull(chars, 13),
GetOrNull(chars, 14), static_cast<char>((chars.size() << 1))} {}
// Returns true if the current instance is empty.
// The 'empty value' is an inlined data value of zero length.
bool is_empty() const { return tag() == 0; }
// Returns true if the current instance holds a tree value.
bool is_tree() const { return (tag() & 1) != 0; }
// Returns true if the current instance holds a cordz_info value.
// Requires the current instance to hold a tree value.
bool is_profiled() const {
assert(is_tree());
return as_tree_.cordz_info != kNullCordzInfo;
}
// Returns the cordz_info sampling instance for this instance, or nullptr
// if the current instance is not sampled and does not have CordzInfo data.
// Requires the current instance to hold a tree value.
CordzInfo* cordz_info() const {
assert(is_tree());
intptr_t info =
static_cast<intptr_t>(absl::big_endian::ToHost64(as_tree_.cordz_info));
assert(info & 1);
return reinterpret_cast<CordzInfo*>(info - 1);
}
// Sets the current cordz_info sampling instance for this instance, or nullptr
// if the current instance is not sampled and does not have CordzInfo data.
// Requires the current instance to hold a tree value.
void set_cordz_info(CordzInfo* cordz_info) {
assert(is_tree());
intptr_t info = reinterpret_cast<intptr_t>(cordz_info) | 1;
as_tree_.cordz_info = absl::big_endian::FromHost64(info);
}
// Resets the current cordz_info to null / empty.
void clear_cordz_info() {
assert(is_tree());
as_tree_.cordz_info = kNullCordzInfo;
}
// Returns a read only pointer to the character data inside this instance.
// Requires the current instance to hold inline data.
const char* as_chars() const {
assert(!is_tree());
return as_chars_;
}
// Returns a mutable pointer to the character data inside this instance.
// Should be used for 'write only' operations setting an inlined value.
// Applications can set the value of inlined data either before or after
// setting the inlined size, i.e., both of the below are valid:
//
// // Set inlined data and inline size
// memcpy(data_.as_chars(), data, size);
// data_.set_inline_size(size);
//
// // Set inlined size and inline data
// data_.set_inline_size(size);
// memcpy(data_.as_chars(), data, size);
//
// It's an error to read from the returned pointer without a preceding write
// if the current instance does not hold inline data, i.e.: is_tree() == true.
char* as_chars() { return as_chars_; }
// Returns the tree value of this value.
// Requires the current instance to hold a tree value.
CordRep* as_tree() const {
assert(is_tree());
return as_tree_.rep;
}
// Initialize this instance to holding the tree value `rep`,
// initializing the cordz_info to null, i.e.: 'not profiled'.
void make_tree(CordRep* rep) {
as_tree_.rep = rep;
as_tree_.cordz_info = kNullCordzInfo;
}
// Set the tree value of this instance to 'rep`.
// Requires the current instance to already hold a tree value.
// Does not affect the value of cordz_info.
void set_tree(CordRep* rep) {
assert(is_tree());
as_tree_.rep = rep;
}
// Returns the size of the inlined character data inside this instance.
// Requires the current instance to hold inline data.
size_t inline_size() const {
assert(!is_tree());
return tag() >> 1;
}
// Sets the size of the inlined character data inside this instance.
// Requires `size` to be <= kMaxInline.
// See the documentation on 'as_chars()' for more information and examples.
void set_inline_size(size_t size) {
ABSL_ASSERT(size <= kMaxInline);
tag() = static_cast<char>(size << 1);
}
// Sets or unsets the 'is_profiled' state of this instance.
// Requires the current instance to hold a tree value.
void set_profiled(bool profiled) {
assert(is_tree());
as_tree_.cordz_info = profiled ? kFakeCordzInfo : kNullCordzInfo;
}
private:
// See cordz_info_t for forced alignment and size of `cordz_info` details.
struct AsTree {
explicit constexpr AsTree(absl::cord_internal::CordRep* tree)
: rep(tree), cordz_info(kNullCordzInfo) {}
// This union uses up extra space so that whether rep is 32 or 64 bits,
// cordz_info will still start at the eighth byte, and the last
// byte of cordz_info will still be the last byte of InlineData.
union {
absl::cord_internal::CordRep* rep;
cordz_info_t unused_aligner;
};
cordz_info_t cordz_info;
};
char& tag() { return reinterpret_cast<char*>(this)[kMaxInline]; }
char tag() const { return reinterpret_cast<const char*>(this)[kMaxInline]; }
// If the data has length <= kMaxInline, we store it in `as_chars_`, and
// store the size in the last char of `as_chars_` shifted left + 1.
// Else we store it in a tree and store a pointer to that tree in
// `as_tree_.rep` and store a tag in `tagged_size`.
union {
char as_chars_[kMaxInline + 1];
AsTree as_tree_;
};
};
static_assert(sizeof(InlineData) == kMaxInline + 1, "");
inline CordRepConcat* CordRep::concat() {
assert(tag == CONCAT);
return static_cast<CordRepConcat*>(this);
}
inline const CordRepConcat* CordRep::concat() const {
assert(tag == CONCAT);
return static_cast<const CordRepConcat*>(this);
}
inline CordRepSubstring* CordRep::substring() {
assert(tag == SUBSTRING);
return static_cast<CordRepSubstring*>(this);
}
inline const CordRepSubstring* CordRep::substring() const {
assert(tag == SUBSTRING);
return static_cast<const CordRepSubstring*>(this);
}
inline CordRepExternal* CordRep::external() {
assert(tag == EXTERNAL);
return static_cast<CordRepExternal*>(this);
}
inline const CordRepExternal* CordRep::external() const {
assert(tag == EXTERNAL);
return static_cast<const CordRepExternal*>(this);
}
inline CordRep* CordRep::Ref(CordRep* rep) {
assert(rep != nullptr);
rep->refcount.Increment();
return rep;
}
inline void CordRep::Unref(CordRep* rep) {
assert(rep != nullptr);
// Expect refcount to be 0. Avoiding the cost of an atomic decrement should
// typically outweigh the cost of an extra branch checking for ref == 1.
if (ABSL_PREDICT_FALSE(!rep->refcount.DecrementExpectHighRefcount())) {
Destroy(rep);
}
}
} // namespace cord_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_