From 96dd2ea63f5ff682848bc7d85ad110c9cdf278b6 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 23 Nov 2022 13:37:58 -0800 Subject: [PATCH] Refactor proto2::Map to move all the key-only operations into a key-only base class. This reduces code duplication. It also enables more generic access for the table-driven parser. PiperOrigin-RevId: 490574869 --- src/google/protobuf/map.h | 1189 ++++++++++++++++-------------- src/google/protobuf/map_test.inc | 25 +- 2 files changed, 653 insertions(+), 561 deletions(-) diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 1484456451..f03f9e6025 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -193,9 +193,19 @@ class MapAllocator { Arena* arena_; }; +// To save on binary size and simplify generic uses of the map types we collapse +// signed/unsigned versions of the same sized integer to the unsigned version. +template +struct KeyForBaseImpl { + using type = T; +}; template -using KeyForTree = std::conditional_t::value, uint64_t, - std::reference_wrapper>; +struct KeyForBaseImpl::value && + std::is_signed::value>> { + using type = std::make_unsigned_t; +}; +template +using KeyForBase = typename KeyForBaseImpl::type; // Default case: Not transparent. // We use std::hash/std::less and all the lookup functions @@ -258,10 +268,46 @@ struct TransparentSupport { }; #endif // defined(__cpp_lib_string_view) +struct NodeBase { + // Align the node to allow KeyNode to predict the location of the key. + // This way sizeof(NodeBase) contains any possible padding it was going to + // have between NodeBase and the key. + alignas(int64_t) alignas(double) alignas(void*) NodeBase* next; +}; + +inline NodeBase* EraseFromLinkedList(NodeBase* item, NodeBase* head) { + if (head == item) { + return head->next; + } else { + head->next = EraseFromLinkedList(item, head->next); + return head; + } +} + +inline bool TableEntryIsTooLong(NodeBase* node) { + const size_t kMaxLength = 8; + size_t count = 0; + do { + ++count; + node = node->next; + } while (node != nullptr); + // Invariant: no linked list ever is more than kMaxLength in length. + GOOGLE_DCHECK_LE(count, kMaxLength); + return count >= kMaxLength; +} + +template +using KeyForTree = std::conditional_t::value, uint64_t, + std::reference_wrapper>; + +template +using LessForTree = typename TransparentSupport< + std::conditional_t::value, uint64_t, T>>::less; + template using TreeForMap = - std::map, void*, typename TransparentSupport::less, - MapAllocator, void*>>>; + std::map, NodeBase*, LessForTree, + MapAllocator, NodeBase*>>>; // Type safe tagged pointer. // We convert to/from nodes and trees using the operations below. @@ -284,13 +330,11 @@ inline bool TableEntryIsList(TableEntryPtr entry) { inline bool TableEntryIsNonEmptyList(TableEntryPtr entry) { return !TableEntryIsEmpty(entry) && TableEntryIsList(entry); } -template -Node* TableEntryToNode(TableEntryPtr entry) { +inline NodeBase* TableEntryToNode(TableEntryPtr entry) { GOOGLE_DCHECK(TableEntryIsList(entry)); - return reinterpret_cast(static_cast(entry)); + return reinterpret_cast(static_cast(entry)); } -template -TableEntryPtr NodeToTableEntry(Node* node) { +inline TableEntryPtr NodeToTableEntry(NodeBase* node) { GOOGLE_DCHECK((reinterpret_cast(node) & 1) == 0); return static_cast(reinterpret_cast(node)); } @@ -362,6 +406,523 @@ size_t SpaceUsedInValues(const Map* map) { inline size_t SpaceUsedInValues(const void*) { return 0; } +// The value might be of different signedness, so use memcpy to extract it. +template ::value, int> = 0> +T ReadKey(const void* ptr) { + T out; + memcpy(&out, ptr, sizeof(T)); + return out; +} + +template ::value, int> = 0> +const T& ReadKey(const void* ptr) { + return *reinterpret_cast(ptr); +} + +template +class KeyMapBase { + using Allocator = internal::MapAllocator; + static_assert(!std::is_signed::value || !std::is_integral::value, + ""); + + public: + using size_type = size_t; + using hasher = typename TransparentSupport::hash; + + explicit constexpr KeyMapBase(Arena* arena) + : num_elements_(0), + num_buckets_(internal::kGlobalEmptyTableSize), + seed_(0), + index_of_first_non_null_(internal::kGlobalEmptyTableSize), + table_(const_cast(internal::kGlobalEmptyTable)), + alloc_(arena) {} + + KeyMapBase(const KeyMapBase&) = delete; + KeyMapBase& operator=(const KeyMapBase&) = delete; + + protected: + enum { kMinTableSize = 8 }; + + struct KeyNode : NodeBase { + static constexpr size_t kOffset = sizeof(NodeBase); + decltype(auto) key() const { + return ReadKey(reinterpret_cast(this) + kOffset); + } + }; + + // Trees. The payload type is a copy of Key, so that we can query the tree + // with Keys that are not in any particular data structure. + // The value is a void* pointing to Node. We use void* instead of Node* to + // avoid code bloat. That way there is only one instantiation of the tree + // class per key type. + using Tree = internal::TreeForMap; + using TreeIterator = typename Tree::iterator; + + class KeyIteratorBase { + public: + // Invariants: + // node_ is always correct. This is handy because the most common + // operations are operator* and operator-> and they only use node_. + // When node_ is set to a non-null value, all the other non-const fields + // are updated to be correct also, but those fields can become stale + // if the underlying map is modified. When those fields are needed they + // are rechecked, and updated if necessary. + KeyIteratorBase() : node_(nullptr), m_(nullptr), bucket_index_(0) {} + + explicit KeyIteratorBase(const KeyMapBase* m) : m_(m) { + SearchFrom(m->index_of_first_non_null_); + } + + KeyIteratorBase(KeyNode* n, const KeyMapBase* m, size_type index) + : node_(n), m_(m), bucket_index_(index) {} + + KeyIteratorBase(TreeIterator tree_it, const KeyMapBase* m, size_type index) + : node_(NodeFromTreeIterator(tree_it)), m_(m), bucket_index_(index) {} + + // Advance through buckets, looking for the first that isn't empty. + // If nothing non-empty is found then leave node_ == nullptr. + void SearchFrom(size_type start_bucket) { + GOOGLE_DCHECK(m_->index_of_first_non_null_ == m_->num_buckets_ || + !m_->TableEntryIsEmpty(m_->index_of_first_non_null_)); + for (size_type i = start_bucket; i < m_->num_buckets_; ++i) { + TableEntryPtr entry = m_->table_[i]; + if (entry == TableEntryPtr{}) continue; + bucket_index_ = i; + if (PROTOBUF_PREDICT_TRUE(internal::TableEntryIsList(entry))) { + node_ = static_cast(TableEntryToNode(entry)); + } else { + Tree* tree = TableEntryToTree(entry); + GOOGLE_DCHECK(!tree->empty()); + node_ = static_cast(tree->begin()->second); + } + return; + } + node_ = nullptr; + bucket_index_ = 0; + } + + friend bool operator==(const KeyIteratorBase& a, const KeyIteratorBase& b) { + return a.node_ == b.node_; + } + friend bool operator!=(const KeyIteratorBase& a, const KeyIteratorBase& b) { + return a.node_ != b.node_; + } + + KeyIteratorBase& operator++() { + if (node_->next == nullptr) { + SearchFrom(bucket_index_ + 1); + } else { + node_ = static_cast(node_->next); + } + return *this; + } + + KeyNode* node_; + const KeyMapBase* m_; + size_type bucket_index_; + }; + + public: + Arena* arena() const { return this->alloc_.arena(); } + + void Swap(KeyMapBase* other) { + std::swap(num_elements_, other->num_elements_); + std::swap(num_buckets_, other->num_buckets_); + std::swap(seed_, other->seed_); + std::swap(index_of_first_non_null_, other->index_of_first_non_null_); + std::swap(table_, other->table_); + std::swap(alloc_, other->alloc_); + } + + hasher hash_function() const { return {}; } + + static size_type max_size() { + return static_cast(1) << (sizeof(void**) >= 8 ? 60 : 28); + } + size_type size() const { return num_elements_; } + bool empty() const { return size() == 0; } + + protected: + PROTOBUF_NOINLINE void erase_no_destroy(size_type b, KeyNode* node) { + TreeIterator tree_it; + const bool is_list = revalidate_if_necessary(b, node, &tree_it); + if (is_list) { + GOOGLE_DCHECK(TableEntryIsNonEmptyList(b)); + auto* head = TableEntryToNode(table_[b]); + head = EraseFromLinkedList(node, head); + table_[b] = NodeToTableEntry(head); + } else { + GOOGLE_DCHECK(this->TableEntryIsTree(b)); + Tree* tree = internal::TableEntryToTree(this->table_[b]); + if (tree_it != tree->begin()) { + auto* prev = std::prev(tree_it)->second; + prev->next = prev->next->next; + } + tree->erase(tree_it); + if (tree->empty()) { + this->DestroyTree(tree); + this->table_[b] = TableEntryPtr{}; + } + } + --num_elements_; + if (PROTOBUF_PREDICT_FALSE(b == index_of_first_non_null_)) { + while (index_of_first_non_null_ < num_buckets_ && + TableEntryIsEmpty(index_of_first_non_null_)) { + ++index_of_first_non_null_; + } + } + } + + struct NodeAndBucket { + NodeBase* node; + size_type bucket; + }; + // TODO(sbenza): We can reduce duplication by coercing `K` to a common type. + // Eg, for string keys we can coerce to string_view. Otherwise, we instantiate + // this with all the different `char[N]` of the caller. + template + NodeAndBucket FindHelper(const K& k, TreeIterator* it = nullptr) const { + size_type b = BucketNumber(k); + if (TableEntryIsNonEmptyList(b)) { + auto* node = internal::TableEntryToNode(table_[b]); + do { + if (internal::TransparentSupport::Equals( + static_cast(node)->key(), k)) { + return {node, b}; + } else { + node = node->next; + } + } while (node != nullptr); + } else if (TableEntryIsTree(b)) { + Tree* tree = internal::TableEntryToTree(table_[b]); + auto tree_it = tree->find(k); + if (it != nullptr) *it = tree_it; + if (tree_it != tree->end()) { + return {tree_it->second, b}; + } + } + return {nullptr, b}; + } + + // Insert the given Node in bucket b. If that would make bucket b too big, + // and bucket b is not a tree, create a tree for buckets b. + // Requires count(*KeyPtrFromNodePtr(node)) == 0 and that b is the correct + // bucket. num_elements_ is not modified. + void InsertUnique(size_type b, KeyNode* node) { + GOOGLE_DCHECK(index_of_first_non_null_ == num_buckets_ || + !TableEntryIsEmpty(index_of_first_non_null_)); + // In practice, the code that led to this point may have already + // determined whether we are inserting into an empty list, a short list, + // or whatever. But it's probably cheap enough to recompute that here; + // it's likely that we're inserting into an empty or short list. + GOOGLE_DCHECK(FindHelper(node->key()).node == nullptr); + if (TableEntryIsEmpty(b)) { + InsertUniqueInList(b, node); + index_of_first_non_null_ = (std::min)(index_of_first_non_null_, b); + } else if (TableEntryIsNonEmptyList(b) && !TableEntryIsTooLong(b)) { + InsertUniqueInList(b, node); + } else { + if (TableEntryIsNonEmptyList(b)) { + TreeConvert(b); + } + GOOGLE_DCHECK(TableEntryIsTree(b)) + << (void*)table_[b] << " " << (uintptr_t)table_[b]; + InsertUniqueInTree(b, node); + index_of_first_non_null_ = (std::min)(index_of_first_non_null_, b); + } + } + + // Returns whether we should insert after the head of the list. For + // non-optimized builds, we randomly decide whether to insert right at the + // head of the list or just after the head. This helps add a little bit of + // non-determinism to the map ordering. + bool ShouldInsertAfterHead(void* node) { +#ifdef NDEBUG + (void)node; + return false; +#else + // Doing modulo with a prime mixes the bits more. + return (reinterpret_cast(node) ^ seed_) % 13 > 6; +#endif + } + + // Helper for InsertUnique. Handles the case where bucket b is a + // not-too-long linked list. + void InsertUniqueInList(size_type b, KeyNode* node) { + if (!TableEntryIsEmpty(b) && ShouldInsertAfterHead(node)) { + auto* first = TableEntryToNode(table_[b]); + node->next = first->next; + first->next = node; + } else { + node->next = TableEntryToNode(table_[b]); + table_[b] = NodeToTableEntry(node); + } + } + + // Helper for InsertUnique. Handles the case where bucket b points to a + // Tree. + void InsertUniqueInTree(size_type b, KeyNode* node) { + auto* tree = TableEntryToTree(table_[b]); + auto it = tree->insert({node->key(), node}).first; + // Maintain the linked list of the nodes in the tree. + // For simplicity, they are in the same order as the tree iteration. + if (it != tree->begin()) { + auto* prev = std::prev(it)->second; + prev->next = node; + } + auto next = std::next(it); + node->next = next != tree->end() ? next->second : nullptr; + } + + // Returns whether it did resize. Currently this is only used when + // num_elements_ increases, though it could be used in other situations. + // It checks for load too low as well as load too high: because any number + // of erases can occur between inserts, the load could be as low as 0 here. + // Resizing to a lower size is not always helpful, but failing to do so can + // destroy the expected big-O bounds for some operations. By having the + // policy that sometimes we resize down as well as up, clients can easily + // keep O(size()) = O(number of buckets) if they want that. + bool ResizeIfLoadIsOutOfRange(size_type new_size) { + const size_type kMaxMapLoadTimes16 = 12; // controls RAM vs CPU tradeoff + const size_type hi_cutoff = num_buckets_ * kMaxMapLoadTimes16 / 16; + const size_type lo_cutoff = hi_cutoff / 4; + // We don't care how many elements are in trees. If a lot are, + // we may resize even though there are many empty buckets. In + // practice, this seems fine. + if (PROTOBUF_PREDICT_FALSE(new_size >= hi_cutoff)) { + if (num_buckets_ <= max_size() / 2) { + Resize(num_buckets_ * 2); + return true; + } + } else if (PROTOBUF_PREDICT_FALSE(new_size <= lo_cutoff && + num_buckets_ > kMinTableSize)) { + size_type lg2_of_size_reduction_factor = 1; + // It's possible we want to shrink a lot here... size() could even be 0. + // So, estimate how much to shrink by making sure we don't shrink so + // much that we would need to grow the table after a few inserts. + const size_type hypothetical_size = new_size * 5 / 4 + 1; + while ((hypothetical_size << lg2_of_size_reduction_factor) < hi_cutoff) { + ++lg2_of_size_reduction_factor; + } + size_type new_num_buckets = std::max( + kMinTableSize, num_buckets_ >> lg2_of_size_reduction_factor); + if (new_num_buckets != num_buckets_) { + Resize(new_num_buckets); + return true; + } + } + return false; + } + + // Resize to the given number of buckets. + void Resize(size_t new_num_buckets) { + if (num_buckets_ == kGlobalEmptyTableSize) { + // This is the global empty array. + // Just overwrite with a new one. No need to transfer or free anything. + num_buckets_ = index_of_first_non_null_ = kMinTableSize; + table_ = CreateEmptyTable(num_buckets_); + seed_ = Seed(); + return; + } + + GOOGLE_DCHECK_GE(new_num_buckets, kMinTableSize); + const auto old_table = table_; + const size_type old_table_size = num_buckets_; + num_buckets_ = new_num_buckets; + table_ = CreateEmptyTable(num_buckets_); + const size_type start = index_of_first_non_null_; + index_of_first_non_null_ = num_buckets_; + for (size_type i = start; i < old_table_size; ++i) { + if (internal::TableEntryIsNonEmptyList(old_table[i])) { + TransferList(static_cast(TableEntryToNode(old_table[i]))); + } else if (internal::TableEntryIsTree(old_table[i])) { + TransferTree(TableEntryToTree(old_table[i])); + } + } + Dealloc(old_table, old_table_size); + } + + // Transfer all nodes in the list `node` into `this`. + void TransferList(KeyNode* node) { + do { + auto* next = static_cast(node->next); + InsertUnique(BucketNumber(node->key()), node); + node = next; + } while (node != nullptr); + } + + // Transfer all nodes in the tree `tree` into `this` and destroy the tree. + void TransferTree(Tree* tree) { + auto* node = tree->begin()->second; + DestroyTree(tree); + TransferList(static_cast(node)); + } + + bool TableEntryIsEmpty(size_type b) const { + return internal::TableEntryIsEmpty(table_[b]); + } + bool TableEntryIsNonEmptyList(size_type b) const { + return internal::TableEntryIsNonEmptyList(table_[b]); + } + bool TableEntryIsTree(size_type b) const { + return internal::TableEntryIsTree(table_[b]); + } + bool TableEntryIsList(size_type b) const { + return internal::TableEntryIsList(table_[b]); + } + + void TreeConvert(size_type b) { + GOOGLE_DCHECK(!TableEntryIsTree(b)); + Tree* tree = + Arena::Create(alloc_.arena(), typename Tree::key_compare(), + typename Tree::allocator_type(alloc_)); + size_type count = CopyListToTree(b, tree); + GOOGLE_DCHECK_EQ(count, tree->size()); + table_[b] = TreeToTableEntry(tree); + // Relink the nodes. + NodeBase* next = nullptr; + auto it = tree->end(); + do { + auto* node = (--it)->second; + node->next = next; + next = node; + } while (it != tree->begin()); + } + + // Copy a linked list in the given bucket to a tree. + // Returns the number of things it copied. + size_type CopyListToTree(size_type b, Tree* tree) { + size_type count = 0; + auto* node = TableEntryToNode(table_[b]); + while (node != nullptr) { + tree->insert({static_cast(node)->key(), node}); + ++count; + auto* next = node->next; + node->next = nullptr; + node = next; + } + return count; + } + + // Return whether table_[b] is a linked list that seems awfully long. + // Requires table_[b] to point to a non-empty linked list. + bool TableEntryIsTooLong(size_type b) { + return internal::TableEntryIsTooLong(TableEntryToNode(table_[b])); + } + + template + size_type BucketNumber(const K& k) const { + // We xor the hash value against the random seed so that we effectively + // have a random hash function. + uint64_t h = hash_function()(k) ^ seed_; + + // We use the multiplication method to determine the bucket number from + // the hash value. The constant kPhi (suggested by Knuth) is roughly + // (sqrt(5) - 1) / 2 * 2^64. + constexpr uint64_t kPhi = uint64_t{0x9e3779b97f4a7c15}; + return ((kPhi * h) >> 32) & (num_buckets_ - 1); + } + + // Return a power of two no less than max(kMinTableSize, n). + // Assumes either n < kMinTableSize or n is a power of two. + size_type TableSize(size_type n) { + return n < static_cast(kMinTableSize) + ? static_cast(kMinTableSize) + : n; + } + + // Use alloc_ to allocate an array of n objects of type U. + template + U* Alloc(size_type n) { + using alloc_type = typename Allocator::template rebind::other; + return alloc_type(alloc_).allocate(n); + } + + // Use alloc_ to deallocate an array of n objects of type U. + template + void Dealloc(U* t, size_type n) { + using alloc_type = typename Allocator::template rebind::other; + alloc_type(alloc_).deallocate(t, n); + } + + void DestroyTree(Tree* tree) { + if (alloc_.arena() == nullptr) { + delete tree; + } + } + + TableEntryPtr* CreateEmptyTable(size_type n) { + GOOGLE_DCHECK(n >= kMinTableSize); + GOOGLE_DCHECK_EQ(n & (n - 1), 0u); + TableEntryPtr* result = Alloc(n); + memset(result, 0, n * sizeof(result[0])); + return result; + } + + // Return a randomish value. + size_type Seed() const { + // We get a little bit of randomness from the address of the map. The + // lower bits are not very random, due to alignment, so we discard them + // and shift the higher bits into their place. + size_type s = reinterpret_cast(this) >> 4; +#if !defined(GOOGLE_PROTOBUF_NO_RDTSC) +#if defined(__APPLE__) + // Use a commpage-based fast time function on Apple environments (MacOS, + // iOS, tvOS, watchOS, etc). + s += mach_absolute_time(); +#elif defined(__x86_64__) && defined(__GNUC__) + uint32_t hi, lo; + asm volatile("rdtsc" : "=a"(lo), "=d"(hi)); + s += ((static_cast(hi) << 32) | lo); +#elif defined(__aarch64__) && defined(__GNUC__) + // There is no rdtsc on ARMv8. CNTVCT_EL0 is the virtual counter of the + // system timer. It runs at a different frequency than the CPU's, but is + // the best source of time-based entropy we get. + uint64_t virtual_timer_value; + asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + s += virtual_timer_value; +#endif +#endif // !defined(GOOGLE_PROTOBUF_NO_RDTSC) + return s; + } + + // Assumes node_ and m_ are correct and non-null, but other fields may be + // stale. Fix them as needed. Then return true iff node_ points to a + // Node in a list. If false is returned then *it is modified to be + // a valid iterator for node_. + bool revalidate_if_necessary(size_t& bucket_index, KeyNode* node, + TreeIterator* it) const { + // Force bucket_index to be in range. + bucket_index &= (num_buckets_ - 1); + // Common case: the bucket we think is relevant points to `node`. + if (table_[bucket_index] == NodeToTableEntry(node)) return true; + // Less common: the bucket is a linked list with node_ somewhere in it, + // but not at the head. + if (TableEntryIsNonEmptyList(bucket_index)) { + auto* l = TableEntryToNode(table_[bucket_index]); + while ((l = l->next) != nullptr) { + if (l == node) { + return true; + } + } + } + // Well, bucket_index_ still might be correct, but probably + // not. Revalidate just to be sure. This case is rare enough that we + // don't worry about potential optimizations, such as having a custom + // find-like method that compares Node* instead of the key. + auto res = FindHelper(node->key(), it); + bucket_index = res.bucket; + return TableEntryIsList(bucket_index); + } + + size_type num_elements_; + size_type num_buckets_; + size_type seed_; + size_type index_of_first_non_null_; + TableEntryPtr* table_; // an array with num_buckets_ entries + Allocator alloc_; +}; + } // namespace internal #ifdef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE @@ -509,46 +1070,34 @@ class Map { // 10. InnerMap uses KeyForTree when using the Tree representation, which // is either `Key`, if Key is a scalar, or `reference_wrapper` // otherwise. This avoids unnecessary copies of string keys, for example. - class InnerMap : private hasher { + class InnerMap : public internal::KeyMapBase> { public: - explicit constexpr InnerMap(Arena* arena) - : hasher(), - num_elements_(0), - num_buckets_(internal::kGlobalEmptyTableSize), - seed_(0), - index_of_first_non_null_(internal::kGlobalEmptyTableSize), - table_(const_cast(internal::kGlobalEmptyTable)), - alloc_(arena) {} + explicit constexpr InnerMap(Arena* arena) : InnerMap::KeyMapBase(arena) {} InnerMap(const InnerMap&) = delete; InnerMap& operator=(const InnerMap&) = delete; ~InnerMap() { - if (alloc_.arena() == nullptr && - num_buckets_ != internal::kGlobalEmptyTableSize) { + if (this->alloc_.arena() == nullptr && + this->num_buckets_ != internal::kGlobalEmptyTableSize) { clear(); - Dealloc(table_, num_buckets_); + this->template Dealloc(this->table_, this->num_buckets_); } } private: - enum { kMinTableSize = 8 }; - // Linked-list nodes, as one would expect for a chaining hash table. - struct Node { + struct Node : InnerMap::KeyMapBase::KeyNode { value_type kv; - Node* next; }; - // Trees. The payload type is a copy of Key, so that we can query the tree - // with Keys that are not in any particular data structure. - // The value is a void* pointing to Node. We use void* instead of Node* to - // avoid code bloat. That way there is only one instantiation of the tree - // class per key type. using Tree = internal::TreeForMap; using TreeIterator = typename Tree::iterator; static Node* NodeFromTreeIterator(TreeIterator it) { + static_assert(PROTOBUF_FIELD_OFFSET(Node, kv.first) == + InnerMap::KeyMapBase::KeyNode::kOffset, + ""); return static_cast(it->second); } @@ -556,181 +1105,69 @@ class Map { // iterator and const_iterator are instantiations of iterator_base. template - class iterator_base { + class iterator_base : public InnerMap::KeyMapBase::KeyIteratorBase { + using Base = typename InnerMap::KeyMapBase::KeyIteratorBase; + public: using reference = KeyValueType&; using pointer = KeyValueType*; - // Invariants: - // node_ is always correct. This is handy because the most common - // operations are operator* and operator-> and they only use node_. - // When node_ is set to a non-null value, all the other non-const fields - // are updated to be correct also, but those fields can become stale - // if the underlying map is modified. When those fields are needed they - // are rechecked, and updated if necessary. - iterator_base() : node_(nullptr), m_(nullptr), bucket_index_(0) {} - - explicit iterator_base(const InnerMap* m) : m_(m) { - SearchFrom(m->index_of_first_non_null_); - } - + using Base::Base; + iterator_base() = default; // Any iterator_base can convert to any other. This is overkill, and we // rely on the enclosing class to use it wisely. The standard "iterator // can convert to const_iterator" is OK but the reverse direction is not. - template - explicit iterator_base(const iterator_base& it) - : node_(it.node_), m_(it.m_), bucket_index_(it.bucket_index_) {} - - iterator_base(Node* n, const InnerMap* m, size_type index) - : node_(n), m_(m), bucket_index_(index) {} - - iterator_base(TreeIterator tree_it, const InnerMap* m, size_type index) - : node_(NodeFromTreeIterator(tree_it)), m_(m), bucket_index_(index) {} - - // Advance through buckets, looking for the first that isn't empty. - // If nothing non-empty is found then leave node_ == nullptr. - void SearchFrom(size_type start_bucket) { - GOOGLE_DCHECK(m_->index_of_first_non_null_ == m_->num_buckets_ || - !m_->TableEntryIsEmpty(m_->index_of_first_non_null_)); - for (size_type i = start_bucket; i < m_->num_buckets_; ++i) { - TableEntryPtr entry = m_->table_[i]; - if (entry == TableEntryPtr{}) continue; - bucket_index_ = i; - if (PROTOBUF_PREDICT_TRUE(internal::TableEntryIsList(entry))) { - node_ = internal::TableEntryToNode(entry); - } else { - Tree* tree = internal::TableEntryToTree(entry); - GOOGLE_DCHECK(!tree->empty()); - node_ = NodeFromTreeIterator(tree->begin()); - } - return; - } - node_ = nullptr; - bucket_index_ = m_->num_buckets_; - } + iterator_base(const Base& base) : Base(base) {} // NOLINT - reference operator*() const { return node_->kv; } - pointer operator->() const { return &(operator*()); } - - friend bool operator==(const iterator_base& a, const iterator_base& b) { - return a.node_ == b.node_; - } - friend bool operator!=(const iterator_base& a, const iterator_base& b) { - return a.node_ != b.node_; - } - - iterator_base& operator++() { - if (node_->next == nullptr) { - SearchFrom(bucket_index_ + 1); - } else { - node_ = node_->next; - } - return *this; - } - - iterator_base operator++(int /* unused */) { - iterator_base tmp = *this; - ++*this; - return tmp; + reference operator*() const { + return static_cast(this->node_)->kv; } - - // Assumes node_ and m_ are correct and non-null, but other fields may be - // stale. Fix them as needed. Then return true iff node_ points to a - // Node in a list. If false is returned then *it is modified to be - // a valid iterator for node_. - bool revalidate_if_necessary(TreeIterator* it) { - GOOGLE_DCHECK(node_ != nullptr && m_ != nullptr); - // Force bucket_index_ to be in range. - bucket_index_ &= (m_->num_buckets_ - 1); - // Common case: the bucket we think is relevant points to node_. - if (m_->table_[bucket_index_] == internal::NodeToTableEntry(node_)) - return true; - // Less common: the bucket is a linked list with node_ somewhere in it, - // but not at the head. - if (m_->TableEntryIsNonEmptyList(bucket_index_)) { - Node* l = internal::TableEntryToNode(m_->table_[bucket_index_]); - while ((l = l->next) != nullptr) { - if (l == node_) { - return true; - } - } - } - // Well, bucket_index_ still might be correct, but probably - // not. Revalidate just to be sure. This case is rare enough that we - // don't worry about potential optimizations, such as having a custom - // find-like method that compares Node* instead of the key. - auto res = m_->FindHelper(node_->kv.first, it); - bucket_index_ = res.bucket; - return m_->TableEntryIsList(bucket_index_); - } - - Node* node_; - const InnerMap* m_; - size_type bucket_index_; + pointer operator->() const { return &(operator*()); } }; public: using iterator = iterator_base; using const_iterator = iterator_base; - Arena* arena() const { return alloc_.arena(); } - - void Swap(InnerMap* other) { - std::swap(num_elements_, other->num_elements_); - std::swap(num_buckets_, other->num_buckets_); - std::swap(seed_, other->seed_); - std::swap(index_of_first_non_null_, other->index_of_first_non_null_); - std::swap(table_, other->table_); - std::swap(alloc_, other->alloc_); - } - iterator begin() { return iterator(this); } iterator end() { return iterator(); } const_iterator begin() const { return const_iterator(this); } const_iterator end() const { return const_iterator(); } void clear() { - for (size_type b = 0; b < num_buckets_; b++) { - Node* node; - if (TableEntryIsNonEmptyList(b)) { - node = internal::TableEntryToNode(table_[b]); - table_[b] = TableEntryPtr{}; - } else if (TableEntryIsTree(b)) { - Tree* tree = internal::TableEntryToTree(table_[b]); - table_[b] = TableEntryPtr{}; + for (size_type b = 0; b < this->num_buckets_; b++) { + internal::NodeBase* node; + if (this->TableEntryIsNonEmptyList(b)) { + node = internal::TableEntryToNode(this->table_[b]); + this->table_[b] = TableEntryPtr{}; + } else if (this->TableEntryIsTree(b)) { + Tree* tree = internal::TableEntryToTree(this->table_[b]); + this->table_[b] = TableEntryPtr{}; node = NodeFromTreeIterator(tree->begin()); - DestroyTree(tree); + this->DestroyTree(tree); } else { continue; } do { - Node* next = node->next; - DestroyNode(node); + auto* next = node->next; + DestroyNode(static_cast(node)); node = next; } while (node != nullptr); } - num_elements_ = 0; - index_of_first_non_null_ = num_buckets_; + this->num_elements_ = 0; + this->index_of_first_non_null_ = this->num_buckets_; } - const hasher& hash_function() const { return *this; } - - static size_type max_size() { - return static_cast(1) << (sizeof(void**) >= 8 ? 60 : 28); - } - size_type size() const { return num_elements_; } - bool empty() const { return size() == 0; } - template iterator find(const K& k) { - auto res = FindHelper(k); - return iterator(res.node, this, res.bucket); + auto res = this->FindHelper(k); + return iterator(static_cast(res.node), this, res.bucket); } template const_iterator find(const K& k) const { - auto res = FindHelper(k); - return const_iterator(res.node, this, res.bucket); + auto res = this->FindHelper(k); + return const_iterator(static_cast(res.node), this, res.bucket); } // Inserts a new element into the container if there is no element with the @@ -761,74 +1198,48 @@ class Map { void erase(iterator it) { GOOGLE_DCHECK_EQ(it.m_, this); - TreeIterator tree_it; - const bool is_list = it.revalidate_if_necessary(&tree_it); - size_type b = it.bucket_index_; - Node* const item = it.node_; - if (is_list) { - GOOGLE_DCHECK(TableEntryIsNonEmptyList(b)); - Node* head = internal::TableEntryToNode(table_[b]); - head = EraseFromLinkedList(item, head); - table_[b] = internal::NodeToTableEntry(head); - } else { - GOOGLE_DCHECK(TableEntryIsTree(b)); - Tree* tree = internal::TableEntryToTree(table_[b]); - if (tree_it != tree->begin()) { - auto* prev = NodeFromTreeIterator(std::prev(tree_it)); - prev->next = prev->next->next; - } - tree->erase(tree_it); - if (tree->empty()) { - DestroyTree(tree); - table_[b] = TableEntryPtr{}; - } - } - DestroyNode(item); - --num_elements_; - if (PROTOBUF_PREDICT_FALSE(b == index_of_first_non_null_)) { - while (index_of_first_non_null_ < num_buckets_ && - TableEntryIsEmpty(index_of_first_non_null_)) { - ++index_of_first_non_null_; - } - } + auto* node = static_cast(it.node_); + this->erase_no_destroy(it.bucket_index_, node); + DestroyNode(node); } size_t SpaceUsedInternal() const { - return internal::SpaceUsedInTable(table_, num_buckets_, - num_elements_, sizeof(Node)); + return internal::SpaceUsedInTable(this->table_, this->num_buckets_, + this->num_elements_, sizeof(Node)); } private: template std::pair TryEmplaceInternal(K&& k, Args&&... args) { - auto p = FindHelper(k); + auto p = this->FindHelper(k); // Case 1: key was already present. if (p.node != nullptr) - return std::make_pair(iterator(p.node, this, p.bucket), false); + return std::make_pair( + iterator(static_cast(p.node), this, p.bucket), false); // Case 2: insert. - if (ResizeIfLoadIsOutOfRange(num_elements_ + 1)) { - p = FindHelper(k); + if (this->ResizeIfLoadIsOutOfRange(this->num_elements_ + 1)) { + p = this->FindHelper(k); } const size_type b = p.bucket; // bucket number // If K is not key_type, make the conversion to key_type explicit. using TypeToInit = typename std::conditional< std::is_same::type, key_type>::value, K&&, key_type>::type; - Node* node = Alloc(1); + Node* node = this->template Alloc(1); // Even when arena is nullptr, CreateInArenaStorage is still used to // ensure the arena of submessage will be consistent. Otherwise, // submessage may have its own arena when message-owned arena is enabled. // Note: This only works if `Key` is not arena constructible. Arena::CreateInArenaStorage(const_cast(&node->kv.first), - alloc_.arena(), + this->alloc_.arena(), static_cast(std::forward(k))); // Note: if `T` is arena constructible, `Args` needs to be empty. - Arena::CreateInArenaStorage(&node->kv.second, alloc_.arena(), + Arena::CreateInArenaStorage(&node->kv.second, this->alloc_.arena(), std::forward(args)...); - iterator result = InsertUnique(b, node); - ++num_elements_; - return std::make_pair(result, true); + this->InsertUnique(b, node); + ++this->num_elements_; + return std::make_pair(iterator(node, this, b), true); } // A helper function to perform an assignment of `mapped_type`. @@ -872,351 +1283,17 @@ class Map { return TryEmplaceInternal(std::forward(args)...); } - struct NodeAndBucket { - Node* node; - size_type bucket; - }; - template - NodeAndBucket FindHelper(const K& k, TreeIterator* it = nullptr) const { - size_type b = BucketNumber(k); - if (TableEntryIsNonEmptyList(b)) { - Node* node = internal::TableEntryToNode(table_[b]); - do { - if (internal::TransparentSupport::Equals(node->kv.first, k)) { - return {node, b}; - } else { - node = node->next; - } - } while (node != nullptr); - } else if (TableEntryIsTree(b)) { - Tree* tree = internal::TableEntryToTree(table_[b]); - auto tree_it = tree->find(k); - if (it != nullptr) *it = tree_it; - if (tree_it != tree->end()) { - return {NodeFromTreeIterator(tree_it), b}; - } - } - return {nullptr, b}; - } - - // Insert the given Node in bucket b. If that would make bucket b too big, - // and bucket b is not a tree, create a tree for buckets b and b^1 to share. - // Requires count(*KeyPtrFromNodePtr(node)) == 0 and that b is the correct - // bucket. num_elements_ is not modified. - iterator InsertUnique(size_type b, Node* node) { - GOOGLE_DCHECK(index_of_first_non_null_ == num_buckets_ || - !TableEntryIsEmpty(index_of_first_non_null_)); - // In practice, the code that led to this point may have already - // determined whether we are inserting into an empty list, a short list, - // or whatever. But it's probably cheap enough to recompute that here; - // it's likely that we're inserting into an empty or short list. - GOOGLE_DCHECK(find(node->kv.first) == end()); - if (TableEntryIsEmpty(b)) { - InsertUniqueInList(b, node); - index_of_first_non_null_ = (std::min)(index_of_first_non_null_, b); - } else if (TableEntryIsNonEmptyList(b) && !TableEntryIsTooLong(b)) { - InsertUniqueInList(b, node); - } else { - if (TableEntryIsNonEmptyList(b)) { - TreeConvert(b); - } - GOOGLE_DCHECK(TableEntryIsTree(b)) - << (void*)table_[b] << " " << (uintptr_t)table_[b]; - InsertUniqueInTree(b, node); - index_of_first_non_null_ = (std::min)(index_of_first_non_null_, b); - } - return iterator(node, this, b); - } - - // Returns whether we should insert after the head of the list. For - // non-optimized builds, we randomly decide whether to insert right at the - // head of the list or just after the head. This helps add a little bit of - // non-determinism to the map ordering. - bool ShouldInsertAfterHead(void* node) { -#ifdef NDEBUG - (void)node; - return false; -#else - // Doing modulo with a prime mixes the bits more. - return (reinterpret_cast(node) ^ seed_) % 13 > 6; -#endif - } - - // Helper for InsertUnique. Handles the case where bucket b is a - // not-too-long linked list. - void InsertUniqueInList(size_type b, Node* node) { - if (!TableEntryIsEmpty(b) && ShouldInsertAfterHead(node)) { - Node* first = internal::TableEntryToNode(table_[b]); - node->next = first->next; - first->next = node; - } else { - node->next = internal::TableEntryToNode(table_[b]); - table_[b] = internal::NodeToTableEntry(node); - } - } - - // Helper for InsertUnique. Handles the case where bucket b points to a - // Tree. - void InsertUniqueInTree(size_type b, Node* node) { - auto* tree = internal::TableEntryToTree(table_[b]); - auto it = tree->insert({node->kv.first, node}).first; - // Maintain the linked list of the nodes in the tree. - // For simplicity, they are in the same order as the tree iteration. - if (it != tree->begin()) { - auto* prev = NodeFromTreeIterator(std::prev(it)); - prev->next = node; - } - auto next = std::next(it); - node->next = next != tree->end() ? NodeFromTreeIterator(next) : nullptr; - } - - // Returns whether it did resize. Currently this is only used when - // num_elements_ increases, though it could be used in other situations. - // It checks for load too low as well as load too high: because any number - // of erases can occur between inserts, the load could be as low as 0 here. - // Resizing to a lower size is not always helpful, but failing to do so can - // destroy the expected big-O bounds for some operations. By having the - // policy that sometimes we resize down as well as up, clients can easily - // keep O(size()) = O(number of buckets) if they want that. - bool ResizeIfLoadIsOutOfRange(size_type new_size) { - const size_type kMaxMapLoadTimes16 = 12; // controls RAM vs CPU tradeoff - const size_type hi_cutoff = num_buckets_ * kMaxMapLoadTimes16 / 16; - const size_type lo_cutoff = hi_cutoff / 4; - // We don't care how many elements are in trees. If a lot are, - // we may resize even though there are many empty buckets. In - // practice, this seems fine. - if (PROTOBUF_PREDICT_FALSE(new_size >= hi_cutoff)) { - if (num_buckets_ <= max_size() / 2) { - Resize(num_buckets_ * 2); - return true; - } - } else if (PROTOBUF_PREDICT_FALSE(new_size <= lo_cutoff && - num_buckets_ > kMinTableSize)) { - size_type lg2_of_size_reduction_factor = 1; - // It's possible we want to shrink a lot here... size() could even be 0. - // So, estimate how much to shrink by making sure we don't shrink so - // much that we would need to grow the table after a few inserts. - const size_type hypothetical_size = new_size * 5 / 4 + 1; - while ((hypothetical_size << lg2_of_size_reduction_factor) < - hi_cutoff) { - ++lg2_of_size_reduction_factor; - } - size_type new_num_buckets = std::max( - kMinTableSize, num_buckets_ >> lg2_of_size_reduction_factor); - if (new_num_buckets != num_buckets_) { - Resize(new_num_buckets); - return true; - } - } - return false; - } - - // Resize to the given number of buckets. - void Resize(size_t new_num_buckets) { - if (num_buckets_ == internal::kGlobalEmptyTableSize) { - // This is the global empty array. - // Just overwrite with a new one. No need to transfer or free anything. - num_buckets_ = index_of_first_non_null_ = kMinTableSize; - table_ = CreateEmptyTable(num_buckets_); - seed_ = Seed(); - return; - } - - GOOGLE_DCHECK_GE(new_num_buckets, kMinTableSize); - const auto old_table = table_; - const size_type old_table_size = num_buckets_; - num_buckets_ = new_num_buckets; - table_ = CreateEmptyTable(num_buckets_); - const size_type start = index_of_first_non_null_; - index_of_first_non_null_ = num_buckets_; - for (size_type i = start; i < old_table_size; ++i) { - if (internal::TableEntryIsNonEmptyList(old_table[i])) { - TransferList(internal::TableEntryToNode(old_table[i])); - } else if (internal::TableEntryIsTree(old_table[i])) { - TransferTree(internal::TableEntryToTree(old_table[i])); - } - } - Dealloc(old_table, old_table_size); - } - - // Transfer all nodes in the list `node` into `this`. - void TransferList(Node* node) { - do { - Node* next = node->next; - InsertUnique(BucketNumber(node->kv.first), node); - node = next; - } while (node != nullptr); - } - - // Transfer all nodes in the tree `tree` into `this` and destroy the tree. - void TransferTree(Tree* tree) { - Node* node = NodeFromTreeIterator(tree->begin()); - DestroyTree(tree); - TransferList(node); - } - - Node* EraseFromLinkedList(Node* item, Node* head) { - if (head == item) { - return head->next; - } else { - head->next = EraseFromLinkedList(item, head->next); - return head; - } - } - - bool TableEntryIsEmpty(size_type b) const { - return internal::TableEntryIsEmpty(table_[b]); - } - bool TableEntryIsNonEmptyList(size_type b) const { - return internal::TableEntryIsNonEmptyList(table_[b]); - } - bool TableEntryIsTree(size_type b) const { - return internal::TableEntryIsTree(table_[b]); - } - bool TableEntryIsList(size_type b) const { - return internal::TableEntryIsList(table_[b]); - } - - void TreeConvert(size_type b) { - GOOGLE_DCHECK(!TableEntryIsTree(b)); - Tree* tree = - Arena::Create(alloc_.arena(), typename Tree::key_compare(), - typename Tree::allocator_type(alloc_)); - size_type count = CopyListToTree(b, tree); - GOOGLE_DCHECK_EQ(count, tree->size()); - table_[b] = internal::TreeToTableEntry(tree); - // Relink the nodes. - Node* next = nullptr; - auto it = tree->end(); - do { - auto* node = NodeFromTreeIterator(--it); - node->next = next; - next = node; - } while (it != tree->begin()); - } - - // Copy a linked list in the given bucket to a tree. - // Returns the number of things it copied. - size_type CopyListToTree(size_type b, Tree* tree) { - size_type count = 0; - Node* node = internal::TableEntryToNode(table_[b]); - while (node != nullptr) { - tree->insert({node->kv.first, node}); - ++count; - Node* next = node->next; - node->next = nullptr; - node = next; - } - return count; - } - - // Return whether table_[b] is a linked list that seems awfully long. - // Requires table_[b] to point to a non-empty linked list. - bool TableEntryIsTooLong(size_type b) { - const size_type kMaxLength = 8; - size_type count = 0; - Node* node = internal::TableEntryToNode(table_[b]); - do { - ++count; - node = node->next; - } while (node != nullptr); - // Invariant: no linked list ever is more than kMaxLength in length. - GOOGLE_DCHECK_LE(count, kMaxLength); - return count >= kMaxLength; - } - - template - size_type BucketNumber(const K& k) const { - // We xor the hash value against the random seed so that we effectively - // have a random hash function. - uint64_t h = hash_function()(k) ^ seed_; - - // We use the multiplication method to determine the bucket number from - // the hash value. The constant kPhi (suggested by Knuth) is roughly - // (sqrt(5) - 1) / 2 * 2^64. - constexpr uint64_t kPhi = uint64_t{0x9e3779b97f4a7c15}; - return ((kPhi * h) >> 32) & (num_buckets_ - 1); - } - - // Return a power of two no less than max(kMinTableSize, n). - // Assumes either n < kMinTableSize or n is a power of two. - size_type TableSize(size_type n) { - return n < static_cast(kMinTableSize) - ? static_cast(kMinTableSize) - : n; - } - - // Use alloc_ to allocate an array of n objects of type U. - template - U* Alloc(size_type n) { - using alloc_type = typename Allocator::template rebind::other; - return alloc_type(alloc_).allocate(n); - } - - // Use alloc_ to deallocate an array of n objects of type U. - template - void Dealloc(U* t, size_type n) { - using alloc_type = typename Allocator::template rebind::other; - alloc_type(alloc_).deallocate(t, n); - } - void DestroyNode(Node* node) { - if (alloc_.arena() == nullptr) { - delete node; + if (this->alloc_.arena() == nullptr) { + node->kv.first.~key_type(); + node->kv.second.~mapped_type(); + this->Dealloc(node, 1); } } - void DestroyTree(Tree* tree) { - if (alloc_.arena() == nullptr) { - delete tree; - } - } - - TableEntryPtr* CreateEmptyTable(size_type n) { - GOOGLE_DCHECK(n >= kMinTableSize); - GOOGLE_DCHECK_EQ(n & (n - 1), 0u); - TableEntryPtr* result = Alloc(n); - memset(result, 0, n * sizeof(result[0])); - return result; - } - - // Return a randomish value. - size_type Seed() const { - // We get a little bit of randomness from the address of the map. The - // lower bits are not very random, due to alignment, so we discard them - // and shift the higher bits into their place. - size_type s = reinterpret_cast(this) >> 4; -#if !defined(GOOGLE_PROTOBUF_NO_RDTSC) -#if defined(__APPLE__) - // Use a commpage-based fast time function on Apple environments (MacOS, - // iOS, tvOS, watchOS, etc). - s += mach_absolute_time(); -#elif defined(__x86_64__) && defined(__GNUC__) - uint32_t hi, lo; - asm volatile("rdtsc" : "=a"(lo), "=d"(hi)); - s += ((static_cast(hi) << 32) | lo); -#elif defined(__aarch64__) && defined(__GNUC__) - // There is no rdtsc on ARMv8. CNTVCT_EL0 is the virtual counter of the - // system timer. It runs at a different frequency than the CPU's, but is - // the best source of time-based entropy we get. - uint64_t virtual_timer_value; - asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); - s += virtual_timer_value; -#endif -#endif // !defined(GOOGLE_PROTOBUF_NO_RDTSC) - return s; - } - friend class Arena; using InternalArenaConstructable_ = void; using DestructorSkippable_ = void; - - size_type num_elements_; - size_type num_buckets_; - size_type seed_; - size_type index_of_first_non_null_; - TableEntryPtr* table_; // an array with num_buckets_ entries - Allocator alloc_; }; // end of class InnerMap template @@ -1245,7 +1322,11 @@ class Map { ++it_; return *this; } - const_iterator operator++(int) { return const_iterator(it_++); } + const_iterator operator++(int) { + auto copy = *this; + ++*this; + return copy; + } friend bool operator==(const const_iterator& a, const const_iterator& b) { return a.it_ == b.it_; @@ -1278,7 +1359,11 @@ class Map { ++it_; return *this; } - iterator operator++(int) { return iterator(it_++); } + iterator operator++(int) { + auto copy = *this; + ++*this; + return copy; + } // Allow implicit conversion to const_iterator. operator const_iterator() const { // NOLINT(runtime/explicit) @@ -1460,9 +1545,7 @@ class Map { void InternalSwap(Map* other) { elements_.Swap(&other->elements_); } - // Access to hasher. Currently this returns a copy, but it may - // be modified to return a const reference in the future. - hasher hash_function() const { return elements_.hash_function(); } + hasher hash_function() const { return {}; } size_t SpaceUsedExcludingSelfLong() const { if (empty()) return 0; diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 824ad3b755..9be584cc87 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -1342,6 +1342,10 @@ TEST_F(MapImplTest, SpaceUsed) { // An newly constructed map should have no space used. EXPECT_EQ(m.SpaceUsedExcludingSelfLong(), 0); + struct IntIntNode : internal::NodeBase { + std::pair kv; + }; + size_t capacity = kMinCap; for (int i = 0; i < 100; ++i) { m[i]; @@ -1349,27 +1353,32 @@ TEST_F(MapImplTest, SpaceUsed) { if (m.size() >= capacity * kMaxLoadFactor) { capacity *= 2; } - EXPECT_EQ( - m.SpaceUsedExcludingSelfLong(), - sizeof(void*) * capacity + - m.size() * sizeof(std::pair, void*>)); + EXPECT_EQ(m.SpaceUsedExcludingSelfLong(), + sizeof(void*) * capacity + m.size() * sizeof(IntIntNode)); } // Test string, and non-scalar keys. Map m2; std::string str = "Some arbitrarily large string"; m2[str] = 1; + + struct StringIntNode : internal::NodeBase { + std::pair kv; + }; + EXPECT_EQ(m2.SpaceUsedExcludingSelfLong(), - sizeof(void*) * kMinCap + - sizeof(std::pair, void*>) + + sizeof(void*) * kMinCap + sizeof(StringIntNode) + internal::StringSpaceUsedExcludingSelfLong(str)); + struct IntAllTypesNode : internal::NodeBase { + std::pair kv; + }; + // Test messages, and non-scalar values. Map m3; m3[0].set_optional_string(str); EXPECT_EQ(m3.SpaceUsedExcludingSelfLong(), - sizeof(void*) * kMinCap + - sizeof(std::pair, void*>) + + sizeof(void*) * kMinCap + sizeof(IntAllTypesNode) + m3[0].SpaceUsedLong() - sizeof(m3[0])); }