From 9c6a50fdd80bb39fabd95faeda84f04062685ff3 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 3 Feb 2021 10:18:15 -0800 Subject: [PATCH] Export of internal Abseil changes -- 4ff721439234e91caf6f7b772e5f554e7dd423c8 by Benjamin Barenblat : Remove endian-sensitivity from hash slow path Prior to this commit, the Abseil hash fast path was endian-agnostic, but the slow path assumed a little-endian platform. Change the slow path to be endian-correct, ensuring that values produced by the fast and slow paths are equal even on big-endian systems. PiperOrigin-RevId: 355424258 -- 7f4fe1aa4de46ad0a2ef19fa9c061fc12a7391ed by Abseil Team : Directly store CordzInfo in the InlineData data contents of InlineRep This greatly reduces the cost of coping and moving cords. Especially the move constructor and move assignment are now back to lean loads and stores without needing any CordzInfo lookups for tracked cords. PiperOrigin-RevId: 355409161 -- 3ca4ca84ed6d98f1e383ffd8d12c28876e905bb3 by Abseil Team : Add #include PiperOrigin-RevId: 355386114 -- 30b0ffad0621971b3135148fcc9e183b0dd2a6bb by Abseil Team : Optimize Cord copy constructor This change avoids double stores of the Cord copy constructor from the zero init of the InlineData / InlineRep contents followed by the assignment and inlines the copy constructor. PiperOrigin-RevId: 355287939 -- 0c043fa7b6e41ca7cefc5edc1e17ad46223e4e77 by CJ Johnson : Now that the absl::Cleanup example returns absl::Status, since we decided on absl::FailedPreconditionError, the precondition should be a positive statement and then the check should be failure to adhere to that positive statement PiperOrigin-RevId: 355216923 -- 9ed922ca5d28fe8790ec6bc0837cf39fbcc92896 by Gennadiy Rozental : Do not set mvsc linker flags for clang-cl (fixes #874) Import of https://github.com/abseil/abseil-cpp/pull/891 PiperOrigin-RevId: 355199380 GitOrigin-RevId: 4ff721439234e91caf6f7b772e5f554e7dd423c8 Change-Id: I3d9d2383549720d7a91f9108dfcd979ad6632fce --- absl/cleanup/cleanup.h | 4 +- absl/cleanup/internal/cleanup.h | 14 ++--- .../internal/unordered_map_constructor_test.h | 1 + absl/hash/BUILD.bazel | 1 + absl/hash/CMakeLists.txt | 1 + absl/hash/internal/hash.h | 51 +++++++++++++++---- absl/strings/cord.cc | 8 +-- absl/strings/cord.h | 28 ++++++++-- absl/strings/cord_test.cc | 4 ++ absl/strings/internal/cord_internal.h | 6 +++ 10 files changed, 90 insertions(+), 28 deletions(-) diff --git a/absl/cleanup/cleanup.h b/absl/cleanup/cleanup.h index 5a4bc546..8ebf1e9b 100644 --- a/absl/cleanup/cleanup.h +++ b/absl/cleanup/cleanup.h @@ -41,7 +41,7 @@ // // Data data; // while (ReadData(source_file, &data)) { -// if (data.IsBad()) { +// if (!data.IsGood()) { // absl::Status result = absl::FailedPreconditionError("Read bad data"); // return result; // Both cleanups execute // } @@ -87,7 +87,7 @@ class ABSL_MUST_USE_RESULT Cleanup { public: Cleanup(Callback callback) // NOLINT - : storage_(std::move(callback), /*engaged=*/true) {} + : storage_(std::move(callback), /* is_callback_engaged = */ true) {} Cleanup(Cleanup&& other) = default; diff --git a/absl/cleanup/internal/cleanup.h b/absl/cleanup/internal/cleanup.h index b68e3dd3..b4c40737 100644 --- a/absl/cleanup/internal/cleanup.h +++ b/absl/cleanup/internal/cleanup.h @@ -45,12 +45,14 @@ class Storage { public: Storage() = delete; - Storage(Callback callback, bool engaged) - : callback_(std::move(callback)), engaged_(engaged) {} + Storage(Callback callback, bool is_callback_engaged) + : callback_(std::move(callback)), + is_callback_engaged_(is_callback_engaged) {} Storage(Storage&& other) : callback_(std::move(other.callback_)), - engaged_(absl::exchange(other.engaged_, false)) {} + is_callback_engaged_( + absl::exchange(other.is_callback_engaged_, false)) {} Storage(const Storage& other) = delete; @@ -58,9 +60,9 @@ class Storage { Storage& operator=(const Storage& other) = delete; - bool IsCallbackEngaged() const { return engaged_; } + bool IsCallbackEngaged() const { return is_callback_engaged_; } - void DisengageCallback() { engaged_ = false; } + void DisengageCallback() { is_callback_engaged_ = false; } void InvokeCallback() ABSL_NO_THREAD_SAFETY_ANALYSIS { std::move(callback_)(); @@ -68,7 +70,7 @@ class Storage { private: Callback callback_; - bool engaged_; + bool is_callback_engaged_; }; } // namespace cleanup_internal diff --git a/absl/container/internal/unordered_map_constructor_test.h b/absl/container/internal/unordered_map_constructor_test.h index 76ee95e6..3f90ad7c 100644 --- a/absl/container/internal/unordered_map_constructor_test.h +++ b/absl/container/internal/unordered_map_constructor_test.h @@ -16,6 +16,7 @@ #define ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_CONSTRUCTOR_TEST_H_ #include +#include #include #include "gmock/gmock.h" diff --git a/absl/hash/BUILD.bazel b/absl/hash/BUILD.bazel index 90c6c8a8..4b2c220f 100644 --- a/absl/hash/BUILD.bazel +++ b/absl/hash/BUILD.bazel @@ -38,6 +38,7 @@ cc_library( deps = [ ":city", ":wyhash", + "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", "//absl/container:fixed_array", diff --git a/absl/hash/CMakeLists.txt b/absl/hash/CMakeLists.txt index 6d198775..b43bfa54 100644 --- a/absl/hash/CMakeLists.txt +++ b/absl/hash/CMakeLists.txt @@ -26,6 +26,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::city + absl::config absl::core_headers absl::endian absl::fixed_array diff --git a/absl/hash/internal/hash.h b/absl/hash/internal/hash.h index eb3471d8..7fb0af0b 100644 --- a/absl/hash/internal/hash.h +++ b/absl/hash/internal/hash.h @@ -38,7 +38,8 @@ #include #include -#include "absl/base/internal/endian.h" +#include "absl/base/config.h" +#include "absl/base/internal/unaligned_access.h" #include "absl/base/port.h" #include "absl/container/fixed_array.h" #include "absl/hash/internal/wyhash.h" @@ -804,26 +805,54 @@ class ABSL_DLL HashState : public HashStateBase { size_t len); // Reads 9 to 16 bytes from p. - // The first 8 bytes are in .first, the rest (zero padded) bytes are in - // .second. + // The least significant 8 bytes are in .first, the rest (zero padded) bytes + // are in .second. static std::pair Read9To16(const unsigned char* p, size_t len) { - uint64_t high = little_endian::Load64(p + len - 8); - return {little_endian::Load64(p), high >> (128 - len * 8)}; + uint64_t low_mem = absl::base_internal::UnalignedLoad64(p); + uint64_t high_mem = absl::base_internal::UnalignedLoad64(p + len - 8); +#ifdef ABSL_IS_LITTLE_ENDIAN + uint64_t most_significant = high_mem; + uint64_t least_significant = low_mem; +#else + uint64_t most_significant = low_mem; + uint64_t least_significant = high_mem; +#endif + return {least_significant, most_significant >> (128 - len * 8)}; } // Reads 4 to 8 bytes from p. Zero pads to fill uint64_t. static uint64_t Read4To8(const unsigned char* p, size_t len) { - return (static_cast(little_endian::Load32(p + len - 4)) - << (len - 4) * 8) | - little_endian::Load32(p); + uint32_t low_mem = absl::base_internal::UnalignedLoad32(p); + uint32_t high_mem = absl::base_internal::UnalignedLoad32(p + len - 4); +#ifdef ABSL_IS_LITTLE_ENDIAN + uint32_t most_significant = high_mem; + uint32_t least_significant = low_mem; +#else + uint32_t most_significant = low_mem; + uint32_t least_significant = high_mem; +#endif + return (static_cast(most_significant) << (len - 4) * 8) | + least_significant; } // Reads 1 to 3 bytes from p. Zero pads to fill uint32_t. static uint32_t Read1To3(const unsigned char* p, size_t len) { - return static_cast((p[0]) | // - (p[len / 2] << (len / 2 * 8)) | // - (p[len - 1] << ((len - 1) * 8))); + unsigned char mem0 = p[0]; + unsigned char mem1 = p[len / 2]; + unsigned char mem2 = p[len - 1]; +#ifdef ABSL_IS_LITTLE_ENDIAN + unsigned char significant2 = mem2; + unsigned char significant1 = mem1; + unsigned char significant0 = mem0; +#else + unsigned char significant2 = mem0; + unsigned char significant1 = mem1; + unsigned char significant0 = mem2; +#endif + return static_cast(significant0 | // + (significant1 << (len / 2 * 8)) | // + (significant2 << ((len - 1) * 8))); } ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Mix(uint64_t state, uint64_t v) { diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index 39191ef5..93533757 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc @@ -495,7 +495,9 @@ void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { data_ = src.data_; if (is_tree()) { + data_.set_profiled(false); CordRep::Ref(tree()); + clear_cordz_info(); } } @@ -509,12 +511,6 @@ void Cord::InlineRep::ClearSlow() { // -------------------------------------------------------------------- // Constructors and destructors -Cord::Cord(const Cord& src) : contents_(src.contents_) { - if (CordRep* tree = contents_.tree()) { - CordRep::Ref(tree); - } -} - Cord::Cord(absl::string_view src) { const size_t n = src.size(); if (n <= InlineRep::kMaxInline) { diff --git a/absl/strings/cord.h b/absl/strings/cord.h index aefb5e53..320226d2 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h @@ -755,6 +755,23 @@ class Cord { bool is_tree() const { return data_.is_tree(); } + // Returns true if the Cord is being profiled by cordz. + bool is_profiled() const { return data_.is_tree() && data_.is_profiled(); } + + // Returns the profiled CordzInfo, or nullptr if not sampled. + absl::cord_internal::CordzInfo* cordz_info() const { + return data_.cordz_info(); + } + + // Sets the profiled CordzInfo. `cordz_info` must not be null. + void set_cordz_info(cord_internal::CordzInfo* cordz_info) { + assert(cordz_info != nullptr); + data_.set_cordz_info(cordz_info); + } + + // Resets the current cordz_info to null / empty. + void clear_cordz_info() { data_.clear_cordz_info(); } + private: friend class Cord; @@ -921,8 +938,12 @@ Cord MakeCordFromExternal(absl::string_view data, Releaser&& releaser) { constexpr Cord::InlineRep::InlineRep(cord_internal::InlineData data) : data_(data) {} -inline Cord::InlineRep::InlineRep(const Cord::InlineRep& src) { - data_ = src.data_; +inline Cord::InlineRep::InlineRep(const Cord::InlineRep& src) + : data_(src.data_) { + if (is_tree()) { + data_.clear_cordz_info(); + absl::cord_internal::CordRep::Ref(as_tree()); + } } inline Cord::InlineRep::InlineRep(Cord::InlineRep&& src) { @@ -956,7 +977,6 @@ inline void Cord::InlineRep::Swap(Cord::InlineRep* rhs) { if (rhs == this) { return; } - std::swap(data_, rhs->data_); } @@ -1037,6 +1057,8 @@ inline Cord& Cord::operator=(const Cord& x) { return *this; } +inline Cord::Cord(const Cord& src) : contents_(src.contents_) {} + inline Cord::Cord(Cord&& src) noexcept : contents_(std::move(src.contents_)) {} inline void Cord::swap(Cord& other) noexcept { diff --git a/absl/strings/cord_test.cc b/absl/strings/cord_test.cc index bf7a6820..f9982428 100644 --- a/absl/strings/cord_test.cc +++ b/absl/strings/cord_test.cc @@ -183,6 +183,10 @@ class CordTestPeer { } static bool IsTree(const Cord& c) { return c.contents_.is_tree(); } + + static cord_internal::CordzInfo* GetCordzInfo(const Cord& c) { + return c.contents_.cordz_info(); + } }; ABSL_NAMESPACE_END diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index cda00a44..a1ba67fe 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h @@ -387,6 +387,12 @@ class InlineData { as_tree_.cordz_info = absl::big_endian::FromHost64(info); } + // Resets the current cordz_info to null / empty. + void clear_cordz_info() { + assert(is_tree()); + as_tree_.cordz_info = kNullCordzInfo; + } + // Returns a read only pointer to the character data inside this instance. // Requires the current instance to hold inline data. const char* as_chars() const {