Export of internal Abseil changes

--
ede5d8e8877c81d7e69549e05076f62cb334ef7f by Abseil Team <absl-team@google.com>:

Optimize 128 bit division in absl::uint128 when the intrinsic does not exist

PiperOrigin-RevId: 316413322

--
5dd02300b5a5700f41e4034b15a1c8c9e7349673 by Abseil Team <absl-team@google.com>:

Add additional 128 bit division benchmarks.

PiperOrigin-RevId: 316358648

--
b7ee6e33b5076b7b6817b728f268d9e58b00b123 by Abseil Team <absl-team@google.com>:

Add tests for ABSL_PREDICT_TRUE and ABSL_PREDICT_FALSE macros.

PiperOrigin-RevId: 316325593

--
3d7fe4ab8bc1e736b9697098eeb2cdb7e9910105 by Andy Soffer <asoffer@google.com>:

Removing unnecessary comment using hurtful words.

PiperOrigin-RevId: 316192184
GitOrigin-RevId: ede5d8e8877c81d7e69549e05076f62cb334ef7f
Change-Id: I4e447286d0b823d99cdd658dd49fb66725bb7a30

absl/base/BUILD.bazel

@@ -799,3 +799,16 @@ cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
+
+cc_test(
+    name = "optimization_test",
+    size = "small",
+    srcs = ["optimization_test.cc"],
+    copts = ABSL_TEST_COPTS,
+    linkopts = ABSL_DEFAULT_LINKOPTS,
+    deps = [
+        ":core_headers",
+        "//absl/types:optional",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

absl/base/CMakeLists.txt

@@ -701,3 +701,15 @@ absl_cc_test(
     absl::fast_type_id
     gtest_main
 )
+
+absl_cc_test(
+  NAME
+    optimization_test
+  SRCS
+    "optimization_test.cc"
+  COPTS
+    ${ABSL_TEST_COPTS}
+  DEPS
+    absl::core_headers
+    gtest_main
+)

absl/base/optimization_test.cc

@@ -0,0 +1,136 @@
+// Copyright 2020 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This test serves primarily as a compilation test for base/raw_logging.h.
+// Raw logging testing is covered by logging_unittest.cc, which is not as
+// portable as this test.
+
+#include "absl/base/optimization.h"
+
+#include "gtest/gtest.h"
+#include "absl/types/optional.h"
+
+namespace {
+
+// Tests for the ABSL_PREDICT_TRUE and ABSL_PREDICT_FALSE macros.
+// The tests only verify that the macros are functionally correct - i.e. code
+// behaves as if they weren't used. They don't try to check their impact on
+// optimization.
+
+TEST(PredictTest, PredictTrue) {
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(true));
+  EXPECT_FALSE(ABSL_PREDICT_TRUE(false));
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(1 == 1));
+  EXPECT_FALSE(ABSL_PREDICT_TRUE(1 == 2));
+
+  if (ABSL_PREDICT_TRUE(false)) ADD_FAILURE();
+  if (!ABSL_PREDICT_TRUE(true)) ADD_FAILURE();
+
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(true) && true);
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(true) || false);
+}
+
+TEST(PredictTest, PredictFalse) {
+  EXPECT_TRUE(ABSL_PREDICT_FALSE(true));
+  EXPECT_FALSE(ABSL_PREDICT_FALSE(false));
+  EXPECT_TRUE(ABSL_PREDICT_FALSE(1 == 1));
+  EXPECT_FALSE(ABSL_PREDICT_FALSE(1 == 2));
+
+  if (ABSL_PREDICT_FALSE(false)) ADD_FAILURE();
+  if (!ABSL_PREDICT_FALSE(true)) ADD_FAILURE();
+
+  EXPECT_TRUE(ABSL_PREDICT_FALSE(true) && true);
+  EXPECT_TRUE(ABSL_PREDICT_FALSE(true) || false);
+}
+
+TEST(PredictTest, OneEvaluation) {
+  // Verify that the expression is only evaluated once.
+  int x = 0;
+  if (ABSL_PREDICT_TRUE((++x) == 0)) ADD_FAILURE();
+  EXPECT_EQ(x, 1);
+  if (ABSL_PREDICT_FALSE((++x) == 0)) ADD_FAILURE();
+  EXPECT_EQ(x, 2);
+}
+
+TEST(PredictTest, OperatorOrder) {
+  // Verify that operator order inside and outside the macro behaves well.
+  // These would fail for a naive '#define ABSL_PREDICT_TRUE(x) x'
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(1 && 2) == true);
+  EXPECT_TRUE(ABSL_PREDICT_FALSE(1 && 2) == true);
+  EXPECT_TRUE(!ABSL_PREDICT_TRUE(1 == 2));
+  EXPECT_TRUE(!ABSL_PREDICT_FALSE(1 == 2));
+}
+
+TEST(PredictTest, Pointer) {
+  const int x = 3;
+  const int *good_intptr = &x;
+  const int *null_intptr = nullptr;
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(good_intptr));
+  EXPECT_FALSE(ABSL_PREDICT_TRUE(null_intptr));
+  // The following doesn't compile:
+  // EXPECT_TRUE(ABSL_PREDICT_FALSE(good_intptr));
+  // EXPECT_FALSE(ABSL_PREDICT_FALSE(null_intptr));
+}
+
+TEST(PredictTest, Optional) {
+  // Note: An optional's truth value is the value's existence, not its truth.
+  absl::optional<bool> has_value(false);
+  absl::optional<bool> no_value;
+  EXPECT_TRUE(ABSL_PREDICT_TRUE(has_value));
+  EXPECT_FALSE(ABSL_PREDICT_TRUE(no_value));
+  // The following doesn't compile:
+  // EXPECT_TRUE(ABSL_PREDICT_FALSE(has_value));
+  // EXPECT_FALSE(ABSL_PREDICT_FALSE(no_value));
+}
+
+class ImplictlyConvertibleToBool {
+ public:
+  explicit ImplictlyConvertibleToBool(bool value) : value_(value) {}
+  operator bool() const {  // NOLINT(google-explicit-constructor)
+    return value_;
+  }
+
+ private:
+  bool value_;
+};
+
+TEST(PredictTest, ImplicitBoolConversion) {
+  const ImplictlyConvertibleToBool is_true(true);
+  const ImplictlyConvertibleToBool is_false(false);
+  if (!ABSL_PREDICT_TRUE(is_true)) ADD_FAILURE();
+  if (ABSL_PREDICT_TRUE(is_false)) ADD_FAILURE();
+  if (!ABSL_PREDICT_FALSE(is_true)) ADD_FAILURE();
+  if (ABSL_PREDICT_FALSE(is_false)) ADD_FAILURE();
+}
+
+class ExplictlyConvertibleToBool {
+ public:
+  explicit ExplictlyConvertibleToBool(bool value) : value_(value) {}
+  explicit operator bool() const { return value_; }
+
+ private:
+  bool value_;
+};
+
+TEST(PredictTest, ExplicitBoolConversion) {
+  const ExplictlyConvertibleToBool is_true(true);
+  const ExplictlyConvertibleToBool is_false(false);
+  if (!ABSL_PREDICT_TRUE(is_true)) ADD_FAILURE();
+  if (ABSL_PREDICT_TRUE(is_false)) ADD_FAILURE();
+  // The following doesn't compile:
+  // if (!ABSL_PREDICT_FALSE(is_true)) ADD_FAILURE();
+  // if (ABSL_PREDICT_FALSE(is_false)) ADD_FAILURE();
+}
+
+}  // namespace

absl/numeric/BUILD.bazel

@@ -35,6 +35,7 @@ cc_library(
     copts = ABSL_DEFAULT_COPTS,
     linkopts = ABSL_DEFAULT_LINKOPTS,
     deps = [
+        "//absl/base:bits",
         "//absl/base:config",
         "//absl/base:core_headers",
     ],

absl/numeric/CMakeLists.txt

@@ -26,6 +26,7 @@ absl_cc_library(
   COPTS
     ${ABSL_DEFAULT_COPTS}
   DEPS
+    absl::bits
     absl::config
     absl::core_headers
   PUBLIC

absl/numeric/int128.cc

@@ -15,6 +15,7 @@
 #include "absl/numeric/int128.h"
 
 #include <stddef.h>
+
 #include <cassert>
 #include <iomanip>
 #include <ostream>  // NOLINT(readability/streams)
@@ -22,6 +23,9 @@
 #include <string>
 #include <type_traits>
 
+#include "absl/base/internal/bits.h"
+#include "absl/base/optimization.h"
+
 namespace absl {
 ABSL_NAMESPACE_BEGIN
 
@@ -31,43 +35,25 @@ ABSL_DLL const uint128 kuint128max = MakeUint128(
 namespace {
 
 // Returns the 0-based position of the last set bit (i.e., most significant bit)
-// in the given uint64_t. The argument may not be 0.
+// in the given uint128. The argument is not 0.
 //
 // For example:
 //   Given: 5 (decimal) == 101 (binary)
 //   Returns: 2
-#define STEP(T, n, pos, sh)                   \
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE int Fls128(uint128 n) {
-  do {                                        \
-    if ((n) >= (static_cast<T>(1) << (sh))) { \
-      (n) = (n) >> (sh);                      \
-      (pos) |= (sh);                          \
-    }                                         \
-  } while (0)
-static inline int Fls64(uint64_t n) {
-  assert(n != 0);
-  int pos = 0;
-  STEP(uint64_t, n, pos, 0x20);
-  uint32_t n32 = static_cast<uint32_t>(n);
-  STEP(uint32_t, n32, pos, 0x10);
-  STEP(uint32_t, n32, pos, 0x08);
-  STEP(uint32_t, n32, pos, 0x04);
-  return pos + ((uint64_t{0x3333333322221100} >> (n32 << 2)) & 0x3);
-}
-#undef STEP
-
-// Like Fls64() above, but returns the 0-based position of the last set bit
-// (i.e., most significant bit) in the given uint128. The argument may not be 0.
-static inline int Fls128(uint128 n) {
   if (uint64_t hi = Uint128High64(n)) {
-    return Fls64(hi) + 64;
+    ABSL_INTERNAL_ASSUME(hi != 0);
+    return 127 - base_internal::CountLeadingZeros64(hi);
   }
-  return Fls64(Uint128Low64(n));
+  const uint64_t low = Uint128Low64(n);
+  ABSL_INTERNAL_ASSUME(low != 0);
+  return 63 - base_internal::CountLeadingZeros64(low);
 }
 
 // Long division/modulo for uint128 implemented using the shift-subtract
 // division algorithm adapted from:
 // https://stackoverflow.com/questions/5386377/division-without-using
-void DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret,
+inline void DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret,
                        uint128* remainder_ret) {
   assert(divisor != 0);
 

absl/numeric/int128_benchmark.cc

@@ -12,15 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "absl/numeric/int128.h"
-
 #include <algorithm>
 #include <cstdint>
+#include <limits>
 #include <random>
 #include <vector>
 
 #include "benchmark/benchmark.h"
 #include "absl/base/config.h"
+#include "absl/numeric/int128.h"
 
 namespace {
 
@@ -32,57 +32,85 @@ std::mt19937 MakeRandomEngine() {
   return std::mt19937(seed);
 }
 
-std::vector<std::pair<absl::uint128, absl::uint128>>
+template <typename T,
-GetRandomClass128SampleUniformDivisor() {
+          typename H = typename std::conditional<
-  std::vector<std::pair<absl::uint128, absl::uint128>> values;
+              std::numeric_limits<T>::is_signed, int64_t, uint64_t>::type>
+std::vector<std::pair<T, T>> GetRandomClass128SampleUniformDivisor() {
+  std::vector<std::pair<T, T>> values;
   std::mt19937 random = MakeRandomEngine();
-  std::uniform_int_distribution<uint64_t> uniform_uint64;
+  std::uniform_int_distribution<H> uniform_h;
   values.reserve(kSampleSize);
   for (size_t i = 0; i < kSampleSize; ++i) {
-    absl::uint128 a =
+    T a{absl::MakeUint128(uniform_h(random), uniform_h(random))};
-        absl::MakeUint128(uniform_uint64(random), uniform_uint64(random));
+    T b{absl::MakeUint128(uniform_h(random), uniform_h(random))};
-    absl::uint128 b =
+    values.emplace_back(std::max(a, b), std::max(T(2), std::min(a, b)));
-        absl::MakeUint128(uniform_uint64(random), uniform_uint64(random));
-    values.emplace_back(std::max(a, b),
-                        std::max(absl::uint128(2), std::min(a, b)));
   }
   return values;
 }
 
+template <typename T>
 void BM_DivideClass128UniformDivisor(benchmark::State& state) {
-  auto values = GetRandomClass128SampleUniformDivisor();
+  auto values = GetRandomClass128SampleUniformDivisor<T>();
   while (state.KeepRunningBatch(values.size())) {
     for (const auto& pair : values) {
       benchmark::DoNotOptimize(pair.first / pair.second);
     }
   }
 }
-BENCHMARK(BM_DivideClass128UniformDivisor);
+BENCHMARK_TEMPLATE(BM_DivideClass128UniformDivisor, absl::uint128);
+BENCHMARK_TEMPLATE(BM_DivideClass128UniformDivisor, absl::int128);
+
+template <typename T>
+void BM_RemainderClass128UniformDivisor(benchmark::State& state) {
+  auto values = GetRandomClass128SampleUniformDivisor<T>();
+  while (state.KeepRunningBatch(values.size())) {
+    for (const auto& pair : values) {
+      benchmark::DoNotOptimize(pair.first % pair.second);
+    }
+  }
+}
+BENCHMARK_TEMPLATE(BM_RemainderClass128UniformDivisor, absl::uint128);
+BENCHMARK_TEMPLATE(BM_RemainderClass128UniformDivisor, absl::int128);
 
-std::vector<std::pair<absl::uint128, uint64_t>>
+template <typename T,
-GetRandomClass128SampleSmallDivisor() {
+          typename H = typename std::conditional<
-  std::vector<std::pair<absl::uint128, uint64_t>> values;
+              std::numeric_limits<T>::is_signed, int64_t, uint64_t>::type>
+std::vector<std::pair<T, H>> GetRandomClass128SampleSmallDivisor() {
+  std::vector<std::pair<T, H>> values;
   std::mt19937 random = MakeRandomEngine();
-  std::uniform_int_distribution<uint64_t> uniform_uint64;
+  std::uniform_int_distribution<H> uniform_h;
   values.reserve(kSampleSize);
   for (size_t i = 0; i < kSampleSize; ++i) {
-    absl::uint128 a =
+    T a{absl::MakeUint128(uniform_h(random), uniform_h(random))};
-        absl::MakeUint128(uniform_uint64(random), uniform_uint64(random));
+    H b{std::max(H{2}, uniform_h(random))};
-    uint64_t b = std::max(uint64_t{2}, uniform_uint64(random));
+    values.emplace_back(std::max(a, T(b)), b);
-    values.emplace_back(std::max(a, absl::uint128(b)), b);
   }
   return values;
 }
 
+template <typename T>
 void BM_DivideClass128SmallDivisor(benchmark::State& state) {
-  auto values = GetRandomClass128SampleSmallDivisor();
+  auto values = GetRandomClass128SampleSmallDivisor<T>();
   while (state.KeepRunningBatch(values.size())) {
     for (const auto& pair : values) {
       benchmark::DoNotOptimize(pair.first / pair.second);
     }
   }
 }
-BENCHMARK(BM_DivideClass128SmallDivisor);
+BENCHMARK_TEMPLATE(BM_DivideClass128SmallDivisor, absl::uint128);
+BENCHMARK_TEMPLATE(BM_DivideClass128SmallDivisor, absl::int128);
+
+template <typename T>
+void BM_RemainderClass128SmallDivisor(benchmark::State& state) {
+  auto values = GetRandomClass128SampleSmallDivisor<T>();
+  while (state.KeepRunningBatch(values.size())) {
+    for (const auto& pair : values) {
+      benchmark::DoNotOptimize(pair.first % pair.second);
+    }
+  }
+}
+BENCHMARK_TEMPLATE(BM_RemainderClass128SmallDivisor, absl::uint128);
+BENCHMARK_TEMPLATE(BM_RemainderClass128SmallDivisor, absl::int128);
 
 std::vector<std::pair<absl::uint128, absl::uint128>> GetRandomClass128Sample() {
   std::vector<std::pair<absl::uint128, absl::uint128>> values;
@@ -121,74 +149,107 @@ BENCHMARK(BM_AddClass128);
 
 // Some implementations of <random> do not support __int128 when it is
 // available, so we make our own uniform_int_distribution-like type.
+template <typename T,
+          typename H = typename std::conditional<
+              std::is_same<T, __int128>::value, int64_t, uint64_t>::type>
 class UniformIntDistribution128 {
  public:
   // NOLINTNEXTLINE: mimicking std::uniform_int_distribution API
-  unsigned __int128 operator()(std::mt19937& generator) {
+  T operator()(std::mt19937& generator) {
-    return (static_cast<unsigned __int128>(dist64_(generator)) << 64) |
+    return (static_cast<T>(dist64_(generator)) << 64) | dist64_(generator);
-           dist64_(generator);
   }
 
  private:
-  std::uniform_int_distribution<uint64_t> dist64_;
+  std::uniform_int_distribution<H> dist64_;
 };
 
-std::vector<std::pair<unsigned __int128, unsigned __int128>>
+template <typename T,
-GetRandomIntrinsic128SampleUniformDivisor() {
+          typename H = typename std::conditional<
-  std::vector<std::pair<unsigned __int128, unsigned __int128>> values;
+              std::is_same<T, __int128>::value, int64_t, uint64_t>::type>
+std::vector<std::pair<T, T>> GetRandomIntrinsic128SampleUniformDivisor() {
+  std::vector<std::pair<T, T>> values;
   std::mt19937 random = MakeRandomEngine();
-  UniformIntDistribution128 uniform_uint128;
+  UniformIntDistribution128<T> uniform_128;
   values.reserve(kSampleSize);
   for (size_t i = 0; i < kSampleSize; ++i) {
-    unsigned __int128 a = uniform_uint128(random);
+    T a = uniform_128(random);
-    unsigned __int128 b = uniform_uint128(random);
+    T b = uniform_128(random);
-    values.emplace_back(
+    values.emplace_back(std::max(a, b),
-        std::max(a, b),
+                        std::max(static_cast<T>(2), std::min(a, b)));
-        std::max(static_cast<unsigned __int128>(2), std::min(a, b)));
   }
   return values;
 }
 
+template <typename T>
 void BM_DivideIntrinsic128UniformDivisor(benchmark::State& state) {
-  auto values = GetRandomIntrinsic128SampleUniformDivisor();
+  auto values = GetRandomIntrinsic128SampleUniformDivisor<T>();
   while (state.KeepRunningBatch(values.size())) {
     for (const auto& pair : values) {
       benchmark::DoNotOptimize(pair.first / pair.second);
     }
   }
 }
-BENCHMARK(BM_DivideIntrinsic128UniformDivisor);
+BENCHMARK_TEMPLATE(BM_DivideIntrinsic128UniformDivisor, unsigned __int128);
+BENCHMARK_TEMPLATE(BM_DivideIntrinsic128UniformDivisor, __int128);
+
+template <typename T>
+void BM_RemainderIntrinsic128UniformDivisor(benchmark::State& state) {
+  auto values = GetRandomIntrinsic128SampleUniformDivisor<T>();
+  while (state.KeepRunningBatch(values.size())) {
+    for (const auto& pair : values) {
+      benchmark::DoNotOptimize(pair.first % pair.second);
+    }
+  }
+}
+BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128UniformDivisor, unsigned __int128);
+BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128UniformDivisor, __int128);
 
-std::vector<std::pair<unsigned __int128, uint64_t>>
+template <typename T,
-GetRandomIntrinsic128SampleSmallDivisor() {
+          typename H = typename std::conditional<
-  std::vector<std::pair<unsigned __int128, uint64_t>> values;
+              std::is_same<T, __int128>::value, int64_t, uint64_t>::type>
+std::vector<std::pair<T, H>> GetRandomIntrinsic128SampleSmallDivisor() {
+  std::vector<std::pair<T, H>> values;
   std::mt19937 random = MakeRandomEngine();
-  UniformIntDistribution128 uniform_uint128;
+  UniformIntDistribution128<T> uniform_int128;
-  std::uniform_int_distribution<uint64_t> uniform_uint64;
+  std::uniform_int_distribution<H> uniform_int64;
   values.reserve(kSampleSize);
   for (size_t i = 0; i < kSampleSize; ++i) {
-    unsigned __int128 a = uniform_uint128(random);
+    T a = uniform_int128(random);
-    uint64_t b = std::max(uint64_t{2}, uniform_uint64(random));
+    H b = std::max(H{2}, uniform_int64(random));
-    values.emplace_back(std::max(a, static_cast<unsigned __int128>(b)), b);
+    values.emplace_back(std::max(a, static_cast<T>(b)), b);
   }
   return values;
 }
 
+template <typename T>
 void BM_DivideIntrinsic128SmallDivisor(benchmark::State& state) {
-  auto values = GetRandomIntrinsic128SampleSmallDivisor();
+  auto values = GetRandomIntrinsic128SampleSmallDivisor<T>();
   while (state.KeepRunningBatch(values.size())) {
     for (const auto& pair : values) {
       benchmark::DoNotOptimize(pair.first / pair.second);
     }
   }
 }
-BENCHMARK(BM_DivideIntrinsic128SmallDivisor);
+BENCHMARK_TEMPLATE(BM_DivideIntrinsic128SmallDivisor, unsigned __int128);
+BENCHMARK_TEMPLATE(BM_DivideIntrinsic128SmallDivisor, __int128);
+
+template <typename T>
+void BM_RemainderIntrinsic128SmallDivisor(benchmark::State& state) {
+  auto values = GetRandomIntrinsic128SampleSmallDivisor<T>();
+  while (state.KeepRunningBatch(values.size())) {
+    for (const auto& pair : values) {
+      benchmark::DoNotOptimize(pair.first % pair.second);
+    }
+  }
+}
+BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128SmallDivisor, unsigned __int128);
+BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128SmallDivisor, __int128);
 
 std::vector<std::pair<unsigned __int128, unsigned __int128>>
       GetRandomIntrinsic128Sample() {
   std::vector<std::pair<unsigned __int128, unsigned __int128>> values;
   std::mt19937 random = MakeRandomEngine();
-  UniformIntDistribution128 uniform_uint128;
+  UniformIntDistribution128<unsigned __int128> uniform_uint128;
   values.reserve(kSampleSize);
   for (size_t i = 0; i < kSampleSize; ++i) {
     values.emplace_back(uniform_uint128(random), uniform_uint128(random));

absl/random/seed_sequences_test.cc

@@ -96,7 +96,6 @@ template <typename URBG>
 void TestReproducibleVariateSequencesForNonsecureURBG() {
   const size_t kNumVariates = 1000;
 
-  // Master RNG instance.
   URBG rng;
   // Reused for both RNG instances.
   auto reusable_seed = absl::CreateSeedSeqFrom(&rng);