Export of internal Abseil changes

--
ede5d8e8877c81d7e69549e05076f62cb334ef7f by Abseil Team <absl-team@google.com>:

Optimize 128 bit division in absl::uint128 when the intrinsic does not exist

PiperOrigin-RevId: 316413322

--
5dd02300b5a5700f41e4034b15a1c8c9e7349673 by Abseil Team <absl-team@google.com>:

Add additional 128 bit division benchmarks.

PiperOrigin-RevId: 316358648

--
b7ee6e33b5076b7b6817b728f268d9e58b00b123 by Abseil Team <absl-team@google.com>:

Add tests for ABSL_PREDICT_TRUE and ABSL_PREDICT_FALSE macros.

PiperOrigin-RevId: 316325593

--
3d7fe4ab8bc1e736b9697098eeb2cdb7e9910105 by Andy Soffer <asoffer@google.com>:

Removing unnecessary comment using hurtful words.

PiperOrigin-RevId: 316192184
GitOrigin-RevId: ede5d8e8877c81d7e69549e05076f62cb334ef7f
Change-Id: I4e447286d0b823d99cdd658dd49fb66725bb7a30
pull/714/head
Abseil Team 5 years ago committed by Andy Getz
parent 2c92bdc7c2
commit 01f5f81f93
  1. 13
      absl/base/BUILD.bazel
  2. 12
      absl/base/CMakeLists.txt
  3. 136
      absl/base/optimization_test.cc
  4. 1
      absl/numeric/BUILD.bazel
  5. 1
      absl/numeric/CMakeLists.txt
  6. 40
      absl/numeric/int128.cc
  7. 161
      absl/numeric/int128_benchmark.cc
  8. 1
      absl/random/seed_sequences_test.cc

@ -799,3 +799,16 @@ cc_test(
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "optimization_test",
size = "small",
srcs = ["optimization_test.cc"],
copts = ABSL_TEST_COPTS,
linkopts = ABSL_DEFAULT_LINKOPTS,
deps = [
":core_headers",
"//absl/types:optional",
"@com_google_googletest//:gtest_main",
],
)

@ -701,3 +701,15 @@ absl_cc_test(
absl::fast_type_id
gtest_main
)
absl_cc_test(
NAME
optimization_test
SRCS
"optimization_test.cc"
COPTS
${ABSL_TEST_COPTS}
DEPS
absl::core_headers
gtest_main
)

@ -0,0 +1,136 @@
// Copyright 2020 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This test serves primarily as a compilation test for base/raw_logging.h.
// Raw logging testing is covered by logging_unittest.cc, which is not as
// portable as this test.
#include "absl/base/optimization.h"
#include "gtest/gtest.h"
#include "absl/types/optional.h"
namespace {
// Tests for the ABSL_PREDICT_TRUE and ABSL_PREDICT_FALSE macros.
// The tests only verify that the macros are functionally correct - i.e. code
// behaves as if they weren't used. They don't try to check their impact on
// optimization.
TEST(PredictTest, PredictTrue) {
EXPECT_TRUE(ABSL_PREDICT_TRUE(true));
EXPECT_FALSE(ABSL_PREDICT_TRUE(false));
EXPECT_TRUE(ABSL_PREDICT_TRUE(1 == 1));
EXPECT_FALSE(ABSL_PREDICT_TRUE(1 == 2));
if (ABSL_PREDICT_TRUE(false)) ADD_FAILURE();
if (!ABSL_PREDICT_TRUE(true)) ADD_FAILURE();
EXPECT_TRUE(ABSL_PREDICT_TRUE(true) && true);
EXPECT_TRUE(ABSL_PREDICT_TRUE(true) || false);
}
TEST(PredictTest, PredictFalse) {
EXPECT_TRUE(ABSL_PREDICT_FALSE(true));
EXPECT_FALSE(ABSL_PREDICT_FALSE(false));
EXPECT_TRUE(ABSL_PREDICT_FALSE(1 == 1));
EXPECT_FALSE(ABSL_PREDICT_FALSE(1 == 2));
if (ABSL_PREDICT_FALSE(false)) ADD_FAILURE();
if (!ABSL_PREDICT_FALSE(true)) ADD_FAILURE();
EXPECT_TRUE(ABSL_PREDICT_FALSE(true) && true);
EXPECT_TRUE(ABSL_PREDICT_FALSE(true) || false);
}
TEST(PredictTest, OneEvaluation) {
// Verify that the expression is only evaluated once.
int x = 0;
if (ABSL_PREDICT_TRUE((++x) == 0)) ADD_FAILURE();
EXPECT_EQ(x, 1);
if (ABSL_PREDICT_FALSE((++x) == 0)) ADD_FAILURE();
EXPECT_EQ(x, 2);
}
TEST(PredictTest, OperatorOrder) {
// Verify that operator order inside and outside the macro behaves well.
// These would fail for a naive '#define ABSL_PREDICT_TRUE(x) x'
EXPECT_TRUE(ABSL_PREDICT_TRUE(1 && 2) == true);
EXPECT_TRUE(ABSL_PREDICT_FALSE(1 && 2) == true);
EXPECT_TRUE(!ABSL_PREDICT_TRUE(1 == 2));
EXPECT_TRUE(!ABSL_PREDICT_FALSE(1 == 2));
}
TEST(PredictTest, Pointer) {
const int x = 3;
const int *good_intptr = &x;
const int *null_intptr = nullptr;
EXPECT_TRUE(ABSL_PREDICT_TRUE(good_intptr));
EXPECT_FALSE(ABSL_PREDICT_TRUE(null_intptr));
// The following doesn't compile:
// EXPECT_TRUE(ABSL_PREDICT_FALSE(good_intptr));
// EXPECT_FALSE(ABSL_PREDICT_FALSE(null_intptr));
}
TEST(PredictTest, Optional) {
// Note: An optional's truth value is the value's existence, not its truth.
absl::optional<bool> has_value(false);
absl::optional<bool> no_value;
EXPECT_TRUE(ABSL_PREDICT_TRUE(has_value));
EXPECT_FALSE(ABSL_PREDICT_TRUE(no_value));
// The following doesn't compile:
// EXPECT_TRUE(ABSL_PREDICT_FALSE(has_value));
// EXPECT_FALSE(ABSL_PREDICT_FALSE(no_value));
}
class ImplictlyConvertibleToBool {
public:
explicit ImplictlyConvertibleToBool(bool value) : value_(value) {}
operator bool() const { // NOLINT(google-explicit-constructor)
return value_;
}
private:
bool value_;
};
TEST(PredictTest, ImplicitBoolConversion) {
const ImplictlyConvertibleToBool is_true(true);
const ImplictlyConvertibleToBool is_false(false);
if (!ABSL_PREDICT_TRUE(is_true)) ADD_FAILURE();
if (ABSL_PREDICT_TRUE(is_false)) ADD_FAILURE();
if (!ABSL_PREDICT_FALSE(is_true)) ADD_FAILURE();
if (ABSL_PREDICT_FALSE(is_false)) ADD_FAILURE();
}
class ExplictlyConvertibleToBool {
public:
explicit ExplictlyConvertibleToBool(bool value) : value_(value) {}
explicit operator bool() const { return value_; }
private:
bool value_;
};
TEST(PredictTest, ExplicitBoolConversion) {
const ExplictlyConvertibleToBool is_true(true);
const ExplictlyConvertibleToBool is_false(false);
if (!ABSL_PREDICT_TRUE(is_true)) ADD_FAILURE();
if (ABSL_PREDICT_TRUE(is_false)) ADD_FAILURE();
// The following doesn't compile:
// if (!ABSL_PREDICT_FALSE(is_true)) ADD_FAILURE();
// if (ABSL_PREDICT_FALSE(is_false)) ADD_FAILURE();
}
} // namespace

@ -35,6 +35,7 @@ cc_library(
copts = ABSL_DEFAULT_COPTS,
linkopts = ABSL_DEFAULT_LINKOPTS,
deps = [
"//absl/base:bits",
"//absl/base:config",
"//absl/base:core_headers",
],

@ -26,6 +26,7 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
absl::bits
absl::config
absl::core_headers
PUBLIC

@ -15,6 +15,7 @@
#include "absl/numeric/int128.h"
#include <stddef.h>
#include <cassert>
#include <iomanip>
#include <ostream> // NOLINT(readability/streams)
@ -22,6 +23,9 @@
#include <string>
#include <type_traits>
#include "absl/base/internal/bits.h"
#include "absl/base/optimization.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
@ -31,44 +35,26 @@ ABSL_DLL const uint128 kuint128max = MakeUint128(
namespace {
// Returns the 0-based position of the last set bit (i.e., most significant bit)
// in the given uint64_t. The argument may not be 0.
// in the given uint128. The argument is not 0.
//
// For example:
// Given: 5 (decimal) == 101 (binary)
// Returns: 2
#define STEP(T, n, pos, sh) \
do { \
if ((n) >= (static_cast<T>(1) << (sh))) { \
(n) = (n) >> (sh); \
(pos) |= (sh); \
} \
} while (0)
static inline int Fls64(uint64_t n) {
assert(n != 0);
int pos = 0;
STEP(uint64_t, n, pos, 0x20);
uint32_t n32 = static_cast<uint32_t>(n);
STEP(uint32_t, n32, pos, 0x10);
STEP(uint32_t, n32, pos, 0x08);
STEP(uint32_t, n32, pos, 0x04);
return pos + ((uint64_t{0x3333333322221100} >> (n32 << 2)) & 0x3);
}
#undef STEP
// Like Fls64() above, but returns the 0-based position of the last set bit
// (i.e., most significant bit) in the given uint128. The argument may not be 0.
static inline int Fls128(uint128 n) {
inline ABSL_ATTRIBUTE_ALWAYS_INLINE int Fls128(uint128 n) {
if (uint64_t hi = Uint128High64(n)) {
return Fls64(hi) + 64;
ABSL_INTERNAL_ASSUME(hi != 0);
return 127 - base_internal::CountLeadingZeros64(hi);
}
return Fls64(Uint128Low64(n));
const uint64_t low = Uint128Low64(n);
ABSL_INTERNAL_ASSUME(low != 0);
return 63 - base_internal::CountLeadingZeros64(low);
}
// Long division/modulo for uint128 implemented using the shift-subtract
// division algorithm adapted from:
// https://stackoverflow.com/questions/5386377/division-without-using
void DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret,
uint128* remainder_ret) {
inline void DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret,
uint128* remainder_ret) {
assert(divisor != 0);
if (divisor > dividend) {

@ -12,15 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/numeric/int128.h"
#include <algorithm>
#include <cstdint>
#include <limits>
#include <random>
#include <vector>
#include "benchmark/benchmark.h"
#include "absl/base/config.h"
#include "absl/numeric/int128.h"
namespace {
@ -32,57 +32,85 @@ std::mt19937 MakeRandomEngine() {
return std::mt19937(seed);
}
std::vector<std::pair<absl::uint128, absl::uint128>>
GetRandomClass128SampleUniformDivisor() {
std::vector<std::pair<absl::uint128, absl::uint128>> values;
template <typename T,
typename H = typename std::conditional<
std::numeric_limits<T>::is_signed, int64_t, uint64_t>::type>
std::vector<std::pair<T, T>> GetRandomClass128SampleUniformDivisor() {
std::vector<std::pair<T, T>> values;
std::mt19937 random = MakeRandomEngine();
std::uniform_int_distribution<uint64_t> uniform_uint64;
std::uniform_int_distribution<H> uniform_h;
values.reserve(kSampleSize);
for (size_t i = 0; i < kSampleSize; ++i) {
absl::uint128 a =
absl::MakeUint128(uniform_uint64(random), uniform_uint64(random));
absl::uint128 b =
absl::MakeUint128(uniform_uint64(random), uniform_uint64(random));
values.emplace_back(std::max(a, b),
std::max(absl::uint128(2), std::min(a, b)));
T a{absl::MakeUint128(uniform_h(random), uniform_h(random))};
T b{absl::MakeUint128(uniform_h(random), uniform_h(random))};
values.emplace_back(std::max(a, b), std::max(T(2), std::min(a, b)));
}
return values;
}
template <typename T>
void BM_DivideClass128UniformDivisor(benchmark::State& state) {
auto values = GetRandomClass128SampleUniformDivisor();
auto values = GetRandomClass128SampleUniformDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first / pair.second);
}
}
}
BENCHMARK(BM_DivideClass128UniformDivisor);
BENCHMARK_TEMPLATE(BM_DivideClass128UniformDivisor, absl::uint128);
BENCHMARK_TEMPLATE(BM_DivideClass128UniformDivisor, absl::int128);
template <typename T>
void BM_RemainderClass128UniformDivisor(benchmark::State& state) {
auto values = GetRandomClass128SampleUniformDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first % pair.second);
}
}
}
BENCHMARK_TEMPLATE(BM_RemainderClass128UniformDivisor, absl::uint128);
BENCHMARK_TEMPLATE(BM_RemainderClass128UniformDivisor, absl::int128);
std::vector<std::pair<absl::uint128, uint64_t>>
GetRandomClass128SampleSmallDivisor() {
std::vector<std::pair<absl::uint128, uint64_t>> values;
template <typename T,
typename H = typename std::conditional<
std::numeric_limits<T>::is_signed, int64_t, uint64_t>::type>
std::vector<std::pair<T, H>> GetRandomClass128SampleSmallDivisor() {
std::vector<std::pair<T, H>> values;
std::mt19937 random = MakeRandomEngine();
std::uniform_int_distribution<uint64_t> uniform_uint64;
std::uniform_int_distribution<H> uniform_h;
values.reserve(kSampleSize);
for (size_t i = 0; i < kSampleSize; ++i) {
absl::uint128 a =
absl::MakeUint128(uniform_uint64(random), uniform_uint64(random));
uint64_t b = std::max(uint64_t{2}, uniform_uint64(random));
values.emplace_back(std::max(a, absl::uint128(b)), b);
T a{absl::MakeUint128(uniform_h(random), uniform_h(random))};
H b{std::max(H{2}, uniform_h(random))};
values.emplace_back(std::max(a, T(b)), b);
}
return values;
}
template <typename T>
void BM_DivideClass128SmallDivisor(benchmark::State& state) {
auto values = GetRandomClass128SampleSmallDivisor();
auto values = GetRandomClass128SampleSmallDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first / pair.second);
}
}
}
BENCHMARK(BM_DivideClass128SmallDivisor);
BENCHMARK_TEMPLATE(BM_DivideClass128SmallDivisor, absl::uint128);
BENCHMARK_TEMPLATE(BM_DivideClass128SmallDivisor, absl::int128);
template <typename T>
void BM_RemainderClass128SmallDivisor(benchmark::State& state) {
auto values = GetRandomClass128SampleSmallDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first % pair.second);
}
}
}
BENCHMARK_TEMPLATE(BM_RemainderClass128SmallDivisor, absl::uint128);
BENCHMARK_TEMPLATE(BM_RemainderClass128SmallDivisor, absl::int128);
std::vector<std::pair<absl::uint128, absl::uint128>> GetRandomClass128Sample() {
std::vector<std::pair<absl::uint128, absl::uint128>> values;
@ -121,74 +149,107 @@ BENCHMARK(BM_AddClass128);
// Some implementations of <random> do not support __int128 when it is
// available, so we make our own uniform_int_distribution-like type.
template <typename T,
typename H = typename std::conditional<
std::is_same<T, __int128>::value, int64_t, uint64_t>::type>
class UniformIntDistribution128 {
public:
// NOLINTNEXTLINE: mimicking std::uniform_int_distribution API
unsigned __int128 operator()(std::mt19937& generator) {
return (static_cast<unsigned __int128>(dist64_(generator)) << 64) |
dist64_(generator);
T operator()(std::mt19937& generator) {
return (static_cast<T>(dist64_(generator)) << 64) | dist64_(generator);
}
private:
std::uniform_int_distribution<uint64_t> dist64_;
std::uniform_int_distribution<H> dist64_;
};
std::vector<std::pair<unsigned __int128, unsigned __int128>>
GetRandomIntrinsic128SampleUniformDivisor() {
std::vector<std::pair<unsigned __int128, unsigned __int128>> values;
template <typename T,
typename H = typename std::conditional<
std::is_same<T, __int128>::value, int64_t, uint64_t>::type>
std::vector<std::pair<T, T>> GetRandomIntrinsic128SampleUniformDivisor() {
std::vector<std::pair<T, T>> values;
std::mt19937 random = MakeRandomEngine();
UniformIntDistribution128 uniform_uint128;
UniformIntDistribution128<T> uniform_128;
values.reserve(kSampleSize);
for (size_t i = 0; i < kSampleSize; ++i) {
unsigned __int128 a = uniform_uint128(random);
unsigned __int128 b = uniform_uint128(random);
values.emplace_back(
std::max(a, b),
std::max(static_cast<unsigned __int128>(2), std::min(a, b)));
T a = uniform_128(random);
T b = uniform_128(random);
values.emplace_back(std::max(a, b),
std::max(static_cast<T>(2), std::min(a, b)));
}
return values;
}
template <typename T>
void BM_DivideIntrinsic128UniformDivisor(benchmark::State& state) {
auto values = GetRandomIntrinsic128SampleUniformDivisor();
auto values = GetRandomIntrinsic128SampleUniformDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first / pair.second);
}
}
}
BENCHMARK(BM_DivideIntrinsic128UniformDivisor);
BENCHMARK_TEMPLATE(BM_DivideIntrinsic128UniformDivisor, unsigned __int128);
BENCHMARK_TEMPLATE(BM_DivideIntrinsic128UniformDivisor, __int128);
template <typename T>
void BM_RemainderIntrinsic128UniformDivisor(benchmark::State& state) {
auto values = GetRandomIntrinsic128SampleUniformDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first % pair.second);
}
}
}
BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128UniformDivisor, unsigned __int128);
BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128UniformDivisor, __int128);
std::vector<std::pair<unsigned __int128, uint64_t>>
GetRandomIntrinsic128SampleSmallDivisor() {
std::vector<std::pair<unsigned __int128, uint64_t>> values;
template <typename T,
typename H = typename std::conditional<
std::is_same<T, __int128>::value, int64_t, uint64_t>::type>
std::vector<std::pair<T, H>> GetRandomIntrinsic128SampleSmallDivisor() {
std::vector<std::pair<T, H>> values;
std::mt19937 random = MakeRandomEngine();
UniformIntDistribution128 uniform_uint128;
std::uniform_int_distribution<uint64_t> uniform_uint64;
UniformIntDistribution128<T> uniform_int128;
std::uniform_int_distribution<H> uniform_int64;
values.reserve(kSampleSize);
for (size_t i = 0; i < kSampleSize; ++i) {
unsigned __int128 a = uniform_uint128(random);
uint64_t b = std::max(uint64_t{2}, uniform_uint64(random));
values.emplace_back(std::max(a, static_cast<unsigned __int128>(b)), b);
T a = uniform_int128(random);
H b = std::max(H{2}, uniform_int64(random));
values.emplace_back(std::max(a, static_cast<T>(b)), b);
}
return values;
}
template <typename T>
void BM_DivideIntrinsic128SmallDivisor(benchmark::State& state) {
auto values = GetRandomIntrinsic128SampleSmallDivisor();
auto values = GetRandomIntrinsic128SampleSmallDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first / pair.second);
}
}
}
BENCHMARK(BM_DivideIntrinsic128SmallDivisor);
BENCHMARK_TEMPLATE(BM_DivideIntrinsic128SmallDivisor, unsigned __int128);
BENCHMARK_TEMPLATE(BM_DivideIntrinsic128SmallDivisor, __int128);
template <typename T>
void BM_RemainderIntrinsic128SmallDivisor(benchmark::State& state) {
auto values = GetRandomIntrinsic128SampleSmallDivisor<T>();
while (state.KeepRunningBatch(values.size())) {
for (const auto& pair : values) {
benchmark::DoNotOptimize(pair.first % pair.second);
}
}
}
BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128SmallDivisor, unsigned __int128);
BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128SmallDivisor, __int128);
std::vector<std::pair<unsigned __int128, unsigned __int128>>
GetRandomIntrinsic128Sample() {
std::vector<std::pair<unsigned __int128, unsigned __int128>> values;
std::mt19937 random = MakeRandomEngine();
UniformIntDistribution128 uniform_uint128;
UniformIntDistribution128<unsigned __int128> uniform_uint128;
values.reserve(kSampleSize);
for (size_t i = 0; i < kSampleSize; ++i) {
values.emplace_back(uniform_uint128(random), uniform_uint128(random));

@ -96,7 +96,6 @@ template <typename URBG>
void TestReproducibleVariateSequencesForNonsecureURBG() {
const size_t kNumVariates = 1000;
// Master RNG instance.
URBG rng;
// Reused for both RNG instances.
auto reusable_seed = absl::CreateSeedSeqFrom(&rng);

Loading…
Cancel
Save