|
|
|
// Copyright 2017 The Abseil Authors.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
#include "absl/time/clock.h"
|
|
|
|
|
|
|
|
#include "absl/base/attributes.h"
|
|
|
|
|
|
|
|
#ifdef _WIN32
|
|
|
|
#include <windows.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
#include <atomic>
|
|
|
|
#include <cerrno>
|
|
|
|
#include <cstdint>
|
|
|
|
#include <ctime>
|
|
|
|
#include <limits>
|
|
|
|
|
|
|
|
#include "absl/base/internal/spinlock.h"
|
|
|
|
#include "absl/base/internal/unscaledcycleclock.h"
|
|
|
|
#include "absl/base/macros.h"
|
|
|
|
#include "absl/base/port.h"
|
|
|
|
#include "absl/base/thread_annotations.h"
|
|
|
|
|
|
|
|
namespace absl {
|
|
|
|
Time Now() {
|
|
|
|
// TODO(bww): Get a timespec instead so we don't have to divide.
|
|
|
|
int64_t n = absl::GetCurrentTimeNanos();
|
|
|
|
if (n >= 0) {
|
|
|
|
return time_internal::FromUnixDuration(
|
|
|
|
time_internal::MakeDuration(n / 1000000000, n % 1000000000 * 4));
|
|
|
|
}
|
|
|
|
return time_internal::FromUnixDuration(absl::Nanoseconds(n));
|
|
|
|
}
|
|
|
|
} // namespace absl
|
|
|
|
|
|
|
|
// Decide if we should use the fast GetCurrentTimeNanos() algorithm
|
|
|
|
// based on the cyclecounter, otherwise just get the time directly
|
|
|
|
// from the OS on every call. This can be chosen at compile-time via
|
|
|
|
// -DABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS=[0|1]
|
|
|
|
#ifndef ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS
|
|
|
|
#if ABSL_USE_UNSCALED_CYCLECLOCK
|
|
|
|
#define ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS 1
|
|
|
|
#else
|
|
|
|
#define ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS 0
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if defined(__APPLE__)
|
|
|
|
#include "absl/time/internal/get_current_time_ios.inc"
|
|
|
|
#elif defined(_WIN32)
|
|
|
|
#include "absl/time/internal/get_current_time_windows.inc"
|
|
|
|
#else
|
|
|
|
#include "absl/time/internal/get_current_time_posix.inc"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
// Allows override by test.
|
|
|
|
#ifndef GET_CURRENT_TIME_NANOS_FROM_SYSTEM
|
|
|
|
#define GET_CURRENT_TIME_NANOS_FROM_SYSTEM() \
|
|
|
|
::absl::time_internal::GetCurrentTimeNanosFromSystem()
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if !ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS
|
|
|
|
namespace absl {
|
|
|
|
int64_t GetCurrentTimeNanos() {
|
|
|
|
return GET_CURRENT_TIME_NANOS_FROM_SYSTEM();
|
|
|
|
}
|
|
|
|
} // namespace absl
|
|
|
|
#else // Use the cyclecounter-based implementation below.
|
|
|
|
|
|
|
|
// Allows override by test.
|
|
|
|
#ifndef GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW
|
|
|
|
#define GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW() \
|
|
|
|
::absl::time_internal::UnscaledCycleClockWrapperForGetCurrentTime::Now()
|
|
|
|
#endif
|
|
|
|
|
|
|
|
// The following counters are used only by the test code.
|
|
|
|
static int64_t stats_initializations;
|
|
|
|
static int64_t stats_reinitializations;
|
|
|
|
static int64_t stats_calibrations;
|
|
|
|
static int64_t stats_slow_paths;
|
|
|
|
static int64_t stats_fast_slow_paths;
|
|
|
|
|
|
|
|
namespace absl {
|
|
|
|
namespace time_internal {
|
|
|
|
// This is a friend wrapper around UnscaledCycleClock::Now()
|
|
|
|
// (needed to access UnscaledCycleClock).
|
|
|
|
class UnscaledCycleClockWrapperForGetCurrentTime {
|
|
|
|
public:
|
|
|
|
static int64_t Now() { return base_internal::UnscaledCycleClock::Now(); }
|
|
|
|
};
|
|
|
|
} // namespace time_internal
|
|
|
|
|
|
|
|
// uint64_t is used in this module to provide an extra bit in multiplications
|
|
|
|
|
|
|
|
// Return the time in ns as told by the kernel interface. Place in *cycleclock
|
|
|
|
// the value of the cycleclock at about the time of the syscall.
|
|
|
|
// This call represents the time base that this module synchronizes to.
|
|
|
|
// Ensures that *cycleclock does not step back by up to (1 << 16) from
|
|
|
|
// last_cycleclock, to discard small backward counter steps. (Larger steps are
|
|
|
|
// assumed to be complete resyncs, which shouldn't happen. If they do, a full
|
|
|
|
// reinitialization of the outer algorithm should occur.)
|
|
|
|
static int64_t GetCurrentTimeNanosFromKernel(uint64_t last_cycleclock,
|
|
|
|
uint64_t *cycleclock) {
|
|
|
|
// We try to read clock values at about the same time as the kernel clock.
|
|
|
|
// This value gets adjusted up or down as estimate of how long that should
|
|
|
|
// take, so we can reject attempts that take unusually long.
|
|
|
|
static std::atomic<uint64_t> approx_syscall_time_in_cycles{10 * 1000};
|
|
|
|
|
|
|
|
uint64_t local_approx_syscall_time_in_cycles = // local copy
|
|
|
|
approx_syscall_time_in_cycles.load(std::memory_order_relaxed);
|
|
|
|
|
|
|
|
int64_t current_time_nanos_from_system;
|
|
|
|
uint64_t before_cycles;
|
|
|
|
uint64_t after_cycles;
|
|
|
|
uint64_t elapsed_cycles;
|
|
|
|
int loops = 0;
|
|
|
|
do {
|
|
|
|
before_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
|
|
|
|
current_time_nanos_from_system = GET_CURRENT_TIME_NANOS_FROM_SYSTEM();
|
|
|
|
after_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
|
|
|
|
// elapsed_cycles is unsigned, so is large on overflow
|
|
|
|
elapsed_cycles = after_cycles - before_cycles;
|
|
|
|
if (elapsed_cycles >= local_approx_syscall_time_in_cycles &&
|
|
|
|
++loops == 20) { // clock changed frequencies? Back off.
|
|
|
|
loops = 0;
|
|
|
|
if (local_approx_syscall_time_in_cycles < 1000 * 1000) {
|
|
|
|
local_approx_syscall_time_in_cycles =
|
|
|
|
(local_approx_syscall_time_in_cycles + 1) << 1;
|
|
|
|
}
|
|
|
|
approx_syscall_time_in_cycles.store(
|
|
|
|
local_approx_syscall_time_in_cycles,
|
|
|
|
std::memory_order_relaxed);
|
|
|
|
}
|
|
|
|
} while (elapsed_cycles >= local_approx_syscall_time_in_cycles ||
|
|
|
|
last_cycleclock - after_cycles < (static_cast<uint64_t>(1) << 16));
|
|
|
|
|
|
|
|
// Number of times in a row we've seen a kernel time call take substantially
|
|
|
|
// less than approx_syscall_time_in_cycles.
|
|
|
|
static std::atomic<uint32_t> seen_smaller{ 0 };
|
|
|
|
|
|
|
|
// Adjust approx_syscall_time_in_cycles to be within a factor of 2
|
|
|
|
// of the typical time to execute one iteration of the loop above.
|
|
|
|
if ((local_approx_syscall_time_in_cycles >> 1) < elapsed_cycles) {
|
|
|
|
// measured time is no smaller than half current approximation
|
|
|
|
seen_smaller.store(0, std::memory_order_relaxed);
|
|
|
|
} else if (seen_smaller.fetch_add(1, std::memory_order_relaxed) >= 3) {
|
|
|
|
// smaller delays several times in a row; reduce approximation by 12.5%
|
|
|
|
const uint64_t new_approximation =
|
|
|
|
local_approx_syscall_time_in_cycles -
|
|
|
|
(local_approx_syscall_time_in_cycles >> 3);
|
|
|
|
approx_syscall_time_in_cycles.store(new_approximation,
|
|
|
|
std::memory_order_relaxed);
|
|
|
|
seen_smaller.store(0, std::memory_order_relaxed);
|
|
|
|
}
|
|
|
|
|
|
|
|
*cycleclock = after_cycles;
|
|
|
|
return current_time_nanos_from_system;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------
|
|
|
|
// An implementation of reader-write locks that use no atomic ops in the read
|
|
|
|
// case. This is a generalization of Lamport's method for reading a multiword
|
|
|
|
// clock. Increment a word on each write acquisition, using the low-order bit
|
|
|
|
// as a spinlock; the word is the high word of the "clock". Readers read the
|
|
|
|
// high word, then all other data, then the high word again, and repeat the
|
|
|
|
// read if the reads of the high words yields different answers, or an odd
|
|
|
|
// value (either case suggests possible interference from a writer).
|
|
|
|
// Here we use a spinlock to ensure only one writer at a time, rather than
|
|
|
|
// spinning on the bottom bit of the word to benefit from SpinLock
|
|
|
|
// spin-delay tuning.
|
|
|
|
|
|
|
|
// Acquire seqlock (*seq) and return the value to be written to unlock.
|
|
|
|
static inline uint64_t SeqAcquire(std::atomic<uint64_t> *seq) {
|
|
|
|
uint64_t x = seq->fetch_add(1, std::memory_order_relaxed);
|
|
|
|
|
|
|
|
// We put a release fence between update to *seq and writes to shared data.
|
|
|
|
// Thus all stores to shared data are effectively release operations and
|
|
|
|
// update to *seq above cannot be re-ordered past any of them. Note that
|
|
|
|
// this barrier is not for the fetch_add above. A release barrier for the
|
|
|
|
// fetch_add would be before it, not after.
|
|
|
|
std::atomic_thread_fence(std::memory_order_release);
|
|
|
|
|
|
|
|
return x + 2; // original word plus 2
|
|
|
|
}
|
|
|
|
|
|
|
|
// Release seqlock (*seq) by writing x to it---a value previously returned by
|
|
|
|
// SeqAcquire.
|
|
|
|
static inline void SeqRelease(std::atomic<uint64_t> *seq, uint64_t x) {
|
|
|
|
// The unlock store to *seq must have release ordering so that all
|
|
|
|
// updates to shared data must finish before this store.
|
|
|
|
seq->store(x, std::memory_order_release); // release lock for readers
|
|
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------
|
|
|
|
|
|
|
|
// "nsscaled" is unit of time equal to a (2**kScale)th of a nanosecond.
|
|
|
|
enum { kScale = 30 };
|
|
|
|
|
|
|
|
// The minimum interval between samples of the time base.
|
|
|
|
// We pick enough time to amortize the cost of the sample,
|
|
|
|
// to get a reasonably accurate cycle counter rate reading,
|
|
|
|
// and not so much that calculations will overflow 64-bits.
|
|
|
|
static const uint64_t kMinNSBetweenSamples = 2000 << 20;
|
|
|
|
|
|
|
|
// We require that kMinNSBetweenSamples shifted by kScale
|
|
|
|
// have at least a bit left over for 64-bit calculations.
|
|
|
|
static_assert(((kMinNSBetweenSamples << (kScale + 1)) >> (kScale + 1)) ==
|
|
|
|
kMinNSBetweenSamples,
|
|
|
|
"cannot represent kMaxBetweenSamplesNSScaled");
|
|
|
|
|
|
|
|
// A reader-writer lock protecting the static locations below.
|
|
|
|
// See SeqAcquire() and SeqRelease() above.
|
|
|
|
static absl::base_internal::SpinLock lock(
|
|
|
|
absl::base_internal::kLinkerInitialized);
|
|
|
|
static std::atomic<uint64_t> seq(0);
|
|
|
|
|
|
|
|
// data from a sample of the kernel's time value
|
|
|
|
struct TimeSampleAtomic {
|
|
|
|
std::atomic<uint64_t> raw_ns; // raw kernel time
|
|
|
|
std::atomic<uint64_t> base_ns; // our estimate of time
|
|
|
|
std::atomic<uint64_t> base_cycles; // cycle counter reading
|
|
|
|
std::atomic<uint64_t> nsscaled_per_cycle; // cycle period
|
|
|
|
// cycles before we'll sample again (a scaled reciprocal of the period,
|
|
|
|
// to avoid a division on the fast path).
|
|
|
|
std::atomic<uint64_t> min_cycles_per_sample;
|
|
|
|
};
|
|
|
|
// Same again, but with non-atomic types
|
|
|
|
struct TimeSample {
|
|
|
|
uint64_t raw_ns; // raw kernel time
|
|
|
|
uint64_t base_ns; // our estimate of time
|
|
|
|
uint64_t base_cycles; // cycle counter reading
|
|
|
|
uint64_t nsscaled_per_cycle; // cycle period
|
|
|
|
uint64_t min_cycles_per_sample; // approx cycles before next sample
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct TimeSampleAtomic last_sample; // the last sample; under seq
|
|
|
|
|
|
|
|
static int64_t GetCurrentTimeNanosSlowPath() ABSL_ATTRIBUTE_COLD;
|
|
|
|
|
|
|
|
// Read the contents of *atomic into *sample.
|
|
|
|
// Each field is read atomically, but to maintain atomicity between fields,
|
|
|
|
// the access must be done under a lock.
|
|
|
|
static void ReadTimeSampleAtomic(const struct TimeSampleAtomic *atomic,
|
|
|
|
struct TimeSample *sample) {
|
|
|
|
sample->base_ns = atomic->base_ns.load(std::memory_order_relaxed);
|
|
|
|
sample->base_cycles = atomic->base_cycles.load(std::memory_order_relaxed);
|
|
|
|
sample->nsscaled_per_cycle =
|
|
|
|
atomic->nsscaled_per_cycle.load(std::memory_order_relaxed);
|
|
|
|
sample->min_cycles_per_sample =
|
|
|
|
atomic->min_cycles_per_sample.load(std::memory_order_relaxed);
|
|
|
|
sample->raw_ns = atomic->raw_ns.load(std::memory_order_relaxed);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Public routine.
|
|
|
|
// Algorithm: We wish to compute real time from a cycle counter. In normal
|
|
|
|
// operation, we construct a piecewise linear approximation to the kernel time
|
|
|
|
// source, using the cycle counter value. The start of each line segment is at
|
|
|
|
// the same point as the end of the last, but may have a different slope (that
|
|
|
|
// is, a different idea of the cycle counter frequency). Every couple of
|
|
|
|
// seconds, the kernel time source is sampled and compared with the current
|
|
|
|
// approximation. A new slope is chosen that, if followed for another couple
|
|
|
|
// of seconds, will correct the error at the current position. The information
|
|
|
|
// for a sample is in the "last_sample" struct. The linear approximation is
|
|
|
|
// estimated_time = last_sample.base_ns +
|
|
|
|
// last_sample.ns_per_cycle * (counter_reading - last_sample.base_cycles)
|
|
|
|
// (ns_per_cycle is actually stored in different units and scaled, to avoid
|
|
|
|
// overflow). The base_ns of the next linear approximation is the
|
|
|
|
// estimated_time using the last approximation; the base_cycles is the cycle
|
|
|
|
// counter value at that time; the ns_per_cycle is the number of ns per cycle
|
|
|
|
// measured since the last sample, but adjusted so that most of the difference
|
|
|
|
// between the estimated_time and the kernel time will be corrected by the
|
|
|
|
// estimated time to the next sample. In normal operation, this algorithm
|
|
|
|
// relies on:
|
|
|
|
// - the cycle counter and kernel time rates not changing a lot in a few
|
|
|
|
// seconds.
|
|
|
|
// - the client calling into the code often compared to a couple of seconds, so
|
|
|
|
// the time to the next correction can be estimated.
|
|
|
|
// Any time ns_per_cycle is not known, a major error is detected, or the
|
|
|
|
// assumption about frequent calls is violated, the implementation returns the
|
|
|
|
// kernel time. It records sufficient data that a linear approximation can
|
|
|
|
// resume a little later.
|
|
|
|
|
|
|
|
int64_t GetCurrentTimeNanos() {
|
|
|
|
// read the data from the "last_sample" struct (but don't need raw_ns yet)
|
|
|
|
// The reads of "seq" and test of the values emulate a reader lock.
|
|
|
|
uint64_t base_ns;
|
|
|
|
uint64_t base_cycles;
|
|
|
|
uint64_t nsscaled_per_cycle;
|
|
|
|
uint64_t min_cycles_per_sample;
|
|
|
|
uint64_t seq_read0;
|
|
|
|
uint64_t seq_read1;
|
|
|
|
|
|
|
|
// If we have enough information to interpolate, the value returned will be
|
|
|
|
// derived from this cycleclock-derived time estimate. On some platforms
|
|
|
|
// (POWER) the function to retrieve this value has enough complexity to
|
|
|
|
// contribute to register pressure - reading it early before initializing
|
|
|
|
// the other pieces of the calculation minimizes spill/restore instructions,
|
|
|
|
// minimizing icache cost.
|
|
|
|
uint64_t now_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
|
|
|
|
|
|
|
|
// Acquire pairs with the barrier in SeqRelease - if this load sees that
|
|
|
|
// store, the shared-data reads necessarily see that SeqRelease's updates
|
|
|
|
// to the same shared data.
|
|
|
|
seq_read0 = seq.load(std::memory_order_acquire);
|
|
|
|
|
|
|
|
base_ns = last_sample.base_ns.load(std::memory_order_relaxed);
|
|
|
|
base_cycles = last_sample.base_cycles.load(std::memory_order_relaxed);
|
|
|
|
nsscaled_per_cycle =
|
|
|
|
last_sample.nsscaled_per_cycle.load(std::memory_order_relaxed);
|
|
|
|
min_cycles_per_sample =
|
|
|
|
last_sample.min_cycles_per_sample.load(std::memory_order_relaxed);
|
|
|
|
|
|
|
|
// This acquire fence pairs with the release fence in SeqAcquire. Since it
|
|
|
|
// is sequenced between reads of shared data and seq_read1, the reads of
|
|
|
|
// shared data are effectively acquiring.
|
|
|
|
std::atomic_thread_fence(std::memory_order_acquire);
|
|
|
|
|
|
|
|
// The shared-data reads are effectively acquire ordered, and the
|
|
|
|
// shared-data writes are effectively release ordered. Therefore if our
|
|
|
|
// shared-data reads see any of a particular update's shared-data writes,
|
|
|
|
// seq_read1 is guaranteed to see that update's SeqAcquire.
|
|
|
|
seq_read1 = seq.load(std::memory_order_relaxed);
|
|
|
|
|
|
|
|
// Fast path. Return if min_cycles_per_sample has not yet elapsed since the
|
|
|
|
// last sample, and we read a consistent sample. The fast path activates
|
|
|
|
// only when min_cycles_per_sample is non-zero, which happens when we get an
|
|
|
|
// estimate for the cycle time. The predicate will fail if now_cycles <
|
|
|
|
// base_cycles, or if some other thread is in the slow path.
|
|
|
|
//
|
|
|
|
// Since we now read now_cycles before base_ns, it is possible for now_cycles
|
|
|
|
// to be less than base_cycles (if we were interrupted between those loads and
|
|
|
|
// last_sample was updated). This is harmless, because delta_cycles will wrap
|
|
|
|
// and report a time much much bigger than min_cycles_per_sample. In that case
|
|
|
|
// we will take the slow path.
|
|
|
|
uint64_t delta_cycles = now_cycles - base_cycles;
|
|
|
|
if (seq_read0 == seq_read1 && (seq_read0 & 1) == 0 &&
|
|
|
|
delta_cycles < min_cycles_per_sample) {
|
|
|
|
return base_ns + ((delta_cycles * nsscaled_per_cycle) >> kScale);
|
|
|
|
}
|
|
|
|
return GetCurrentTimeNanosSlowPath();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return (a << kScale)/b.
|
|
|
|
// Zero is returned if b==0. Scaling is performed internally to
|
|
|
|
// preserve precision without overflow.
|
|
|
|
static uint64_t SafeDivideAndScale(uint64_t a, uint64_t b) {
|
|
|
|
// Find maximum safe_shift so that
|
|
|
|
// 0 <= safe_shift <= kScale and (a << safe_shift) does not overflow.
|
|
|
|
int safe_shift = kScale;
|
|
|
|
while (((a << safe_shift) >> safe_shift) != a) {
|
|
|
|
safe_shift--;
|
|
|
|
}
|
|
|
|
uint64_t scaled_b = b >> (kScale - safe_shift);
|
|
|
|
uint64_t quotient = 0;
|
|
|
|
if (scaled_b != 0) {
|
|
|
|
quotient = (a << safe_shift) / scaled_b;
|
|
|
|
}
|
|
|
|
return quotient;
|
|
|
|
}
|
|
|
|
|
|
|
|
static uint64_t UpdateLastSample(
|
|
|
|
uint64_t now_cycles, uint64_t now_ns, uint64_t delta_cycles,
|
|
|
|
const struct TimeSample *sample) ABSL_ATTRIBUTE_COLD;
|
|
|
|
|
|
|
|
// The slow path of GetCurrentTimeNanos(). This is taken while gathering
|
|
|
|
// initial samples, when enough time has elapsed since the last sample, and if
|
|
|
|
// any other thread is writing to last_sample.
|
|
|
|
//
|
|
|
|
// Manually mark this 'noinline' to minimize stack frame size of the fast
|
|
|
|
// path. Without this, sometimes a compiler may inline this big block of code
|
|
|
|
// into the fast past. That causes lots of register spills and reloads that
|
|
|
|
// are unnecessary unless the slow path is taken.
|
|
|
|
//
|
|
|
|
// TODO(absl-team): Remove this attribute when our compiler is smart enough
|
|
|
|
// to do the right thing.
|
|
|
|
ABSL_ATTRIBUTE_NOINLINE
|
|
|
|
static int64_t GetCurrentTimeNanosSlowPath() LOCKS_EXCLUDED(lock) {
|
|
|
|
// Serialize access to slow-path. Fast-path readers are not blocked yet, and
|
|
|
|
// code below must not modify last_sample until the seqlock is acquired.
|
|
|
|
lock.Lock();
|
|
|
|
|
|
|
|
// Sample the kernel time base. This is the definition of
|
|
|
|
// "now" if we take the slow path.
|
|
|
|
static uint64_t last_now_cycles; // protected by lock
|
|
|
|
uint64_t now_cycles;
|
|
|
|
uint64_t now_ns = GetCurrentTimeNanosFromKernel(last_now_cycles, &now_cycles);
|
|
|
|
last_now_cycles = now_cycles;
|
|
|
|
|
|
|
|
uint64_t estimated_base_ns;
|
|
|
|
|
|
|
|
// ----------
|
|
|
|
// Read the "last_sample" values again; this time holding the write lock.
|
|
|
|
struct TimeSample sample;
|
|
|
|
ReadTimeSampleAtomic(&last_sample, &sample);
|
|
|
|
|
|
|
|
// ----------
|
|
|
|
// Try running the fast path again; another thread may have updated the
|
|
|
|
// sample between our run of the fast path and the sample we just read.
|
|
|
|
uint64_t delta_cycles = now_cycles - sample.base_cycles;
|
|
|
|
if (delta_cycles < sample.min_cycles_per_sample) {
|
|
|
|
// Another thread updated the sample. This path does not take the seqlock
|
|
|
|
// so that blocked readers can make progress without blocking new readers.
|
|
|
|
estimated_base_ns = sample.base_ns +
|
|
|
|
((delta_cycles * sample.nsscaled_per_cycle) >> kScale);
|
|
|
|
stats_fast_slow_paths++;
|
|
|
|
} else {
|
|
|
|
estimated_base_ns =
|
|
|
|
UpdateLastSample(now_cycles, now_ns, delta_cycles, &sample);
|
|
|
|
}
|
|
|
|
|
|
|
|
lock.Unlock();
|
|
|
|
|
|
|
|
return estimated_base_ns;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Main part of the algorithm. Locks out readers, updates the approximation
|
|
|
|
// using the new sample from the kernel, and stores the result in last_sample
|
|
|
|
// for readers. Returns the new estimated time.
|
|
|
|
static uint64_t UpdateLastSample(uint64_t now_cycles, uint64_t now_ns,
|
|
|
|
uint64_t delta_cycles,
|
|
|
|
const struct TimeSample *sample)
|
|
|
|
EXCLUSIVE_LOCKS_REQUIRED(lock) {
|
|
|
|
uint64_t estimated_base_ns = now_ns;
|
|
|
|
uint64_t lock_value = SeqAcquire(&seq); // acquire seqlock to block readers
|
|
|
|
|
|
|
|
// The 5s in the next if-statement limits the time for which we will trust
|
|
|
|
// the cycle counter and our last sample to give a reasonable result.
|
|
|
|
// Errors in the rate of the source clock can be multiplied by the ratio
|
|
|
|
// between this limit and kMinNSBetweenSamples.
|
|
|
|
if (sample->raw_ns == 0 || // no recent sample, or clock went backwards
|
|
|
|
sample->raw_ns + static_cast<uint64_t>(5) * 1000 * 1000 * 1000 < now_ns ||
|
|
|
|
now_ns < sample->raw_ns || now_cycles < sample->base_cycles) {
|
|
|
|
// record this sample, and forget any previously known slope.
|
|
|
|
last_sample.raw_ns.store(now_ns, std::memory_order_relaxed);
|
|
|
|
last_sample.base_ns.store(estimated_base_ns, std::memory_order_relaxed);
|
|
|
|
last_sample.base_cycles.store(now_cycles, std::memory_order_relaxed);
|
|
|
|
last_sample.nsscaled_per_cycle.store(0, std::memory_order_relaxed);
|
|
|
|
last_sample.min_cycles_per_sample.store(0, std::memory_order_relaxed);
|
|
|
|
stats_initializations++;
|
|
|
|
} else if (sample->raw_ns + 500 * 1000 * 1000 < now_ns &&
|
|
|
|
sample->base_cycles + 100 < now_cycles) {
|
|
|
|
// Enough time has passed to compute the cycle time.
|
|
|
|
if (sample->nsscaled_per_cycle != 0) { // Have a cycle time estimate.
|
|
|
|
// Compute time from counter reading, but avoiding overflow
|
|
|
|
// delta_cycles may be larger than on the fast path.
|
|
|
|
uint64_t estimated_scaled_ns;
|
|
|
|
int s = -1;
|
|
|
|
do {
|
|
|
|
s++;
|
|
|
|
estimated_scaled_ns = (delta_cycles >> s) * sample->nsscaled_per_cycle;
|
|
|
|
} while (estimated_scaled_ns / sample->nsscaled_per_cycle !=
|
|
|
|
(delta_cycles >> s));
|
|
|
|
estimated_base_ns = sample->base_ns +
|
|
|
|
(estimated_scaled_ns >> (kScale - s));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Compute the assumed cycle time kMinNSBetweenSamples ns into the future
|
|
|
|
// assuming the cycle counter rate stays the same as the last interval.
|
|
|
|
uint64_t ns = now_ns - sample->raw_ns;
|
|
|
|
uint64_t measured_nsscaled_per_cycle = SafeDivideAndScale(ns, delta_cycles);
|
|
|
|
|
|
|
|
uint64_t assumed_next_sample_delta_cycles =
|
|
|
|
SafeDivideAndScale(kMinNSBetweenSamples, measured_nsscaled_per_cycle);
|
|
|
|
|
|
|
|
int64_t diff_ns = now_ns - estimated_base_ns; // estimate low by this much
|
|
|
|
|
|
|
|
// We want to set nsscaled_per_cycle so that our estimate of the ns time
|
|
|
|
// at the assumed cycle time is the assumed ns time.
|
|
|
|
// That is, we want to set nsscaled_per_cycle so:
|
|
|
|
// kMinNSBetweenSamples + diff_ns ==
|
|
|
|
// (assumed_next_sample_delta_cycles * nsscaled_per_cycle) >> kScale
|
|
|
|
// But we wish to damp oscillations, so instead correct only most
|
|
|
|
// of our current error, by solving:
|
|
|
|
// kMinNSBetweenSamples + diff_ns - (diff_ns / 16) ==
|
|
|
|
// (assumed_next_sample_delta_cycles * nsscaled_per_cycle) >> kScale
|
|
|
|
ns = kMinNSBetweenSamples + diff_ns - (diff_ns / 16);
|
|
|
|
uint64_t new_nsscaled_per_cycle =
|
|
|
|
SafeDivideAndScale(ns, assumed_next_sample_delta_cycles);
|
|
|
|
if (new_nsscaled_per_cycle != 0 &&
|
|
|
|
diff_ns < 100 * 1000 * 1000 && -diff_ns < 100 * 1000 * 1000) {
|
|
|
|
// record the cycle time measurement
|
|
|
|
last_sample.nsscaled_per_cycle.store(
|
|
|
|
new_nsscaled_per_cycle, std::memory_order_relaxed);
|
|
|
|
uint64_t new_min_cycles_per_sample =
|
|
|
|
SafeDivideAndScale(kMinNSBetweenSamples, new_nsscaled_per_cycle);
|
|
|
|
last_sample.min_cycles_per_sample.store(
|
|
|
|
new_min_cycles_per_sample, std::memory_order_relaxed);
|
|
|
|
stats_calibrations++;
|
|
|
|
} else { // something went wrong; forget the slope
|
|
|
|
last_sample.nsscaled_per_cycle.store(0, std::memory_order_relaxed);
|
|
|
|
last_sample.min_cycles_per_sample.store(0, std::memory_order_relaxed);
|
|
|
|
estimated_base_ns = now_ns;
|
|
|
|
stats_reinitializations++;
|
|
|
|
}
|
|
|
|
last_sample.raw_ns.store(now_ns, std::memory_order_relaxed);
|
|
|
|
last_sample.base_ns.store(estimated_base_ns, std::memory_order_relaxed);
|
|
|
|
last_sample.base_cycles.store(now_cycles, std::memory_order_relaxed);
|
|
|
|
} else {
|
|
|
|
// have a sample, but no slope; waiting for enough time for a calibration
|
|
|
|
stats_slow_paths++;
|
|
|
|
}
|
|
|
|
|
|
|
|
SeqRelease(&seq, lock_value); // release the readers
|
|
|
|
|
|
|
|
return estimated_base_ns;
|
|
|
|
}
|
|
|
|
} // namespace absl
|
|
|
|
#endif // ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS
|
|
|
|
|
|
|
|
namespace absl {
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
// Returns the maximum duration that SleepOnce() can sleep for.
|
|
|
|
constexpr absl::Duration MaxSleep() {
|
|
|
|
#ifdef _WIN32
|
|
|
|
// Windows Sleep() takes unsigned long argument in milliseconds.
|
|
|
|
return absl::Milliseconds(
|
|
|
|
std::numeric_limits<unsigned long>::max()); // NOLINT(runtime/int)
|
|
|
|
#else
|
|
|
|
return absl::Seconds(std::numeric_limits<time_t>::max());
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
// Sleeps for the given duration.
|
|
|
|
// REQUIRES: to_sleep <= MaxSleep().
|
|
|
|
void SleepOnce(absl::Duration to_sleep) {
|
|
|
|
#ifdef _WIN32
|
|
|
|
Sleep(to_sleep / absl::Milliseconds(1));
|
|
|
|
#else
|
|
|
|
struct timespec sleep_time = absl::ToTimespec(to_sleep);
|
|
|
|
while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) {
|
|
|
|
// Ignore signals and wait for the full interval to elapse.
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
} // namespace absl
|
|
|
|
|
|
|
|
extern "C" {
|
|
|
|
|
|
|
|
ABSL_ATTRIBUTE_WEAK void AbslInternalSleepFor(absl::Duration duration) {
|
|
|
|
while (duration > absl::ZeroDuration()) {
|
|
|
|
absl::Duration to_sleep = std::min(duration, absl::MaxSleep());
|
|
|
|
absl::SleepOnce(to_sleep);
|
|
|
|
duration -= to_sleep;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} // extern "C"
|