Report contended absl::call_once waits via SubmitLockWait. CallOnceImpl already uses SpinLockWait when initialization is in progress on another thread. Expose SpinLock's lock-wait reporting helper as base_internal::SubmitLockWait (keeping the TSan mutex ignore-region diversion in SpinLock::SlowLock) and measure CycleClock::Now() around SpinLockWait on the contended slow path so RegisterSpinLockLockProfiler captures contended absl::call_once and LowLevelCallOnce waits without adding overhead to the initialized fast path or uncontended initialization. PiperOrigin-RevId: 1005937528
diff --git a/absl/base/call_once.h b/absl/base/call_once.h index 58d5d23..f44ee38 100644 --- a/absl/base/call_once.h +++ b/absl/base/call_once.h
@@ -35,9 +35,11 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/cycleclock.h" #include "absl/base/internal/low_level_scheduling.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/spinlock.h" #include "absl/base/internal/spinlock_wait.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" @@ -178,10 +180,17 @@ // kOnceDone. If it returns kOnceDone, it must have loaded the control word // with std::memory_order_acquire and seen a value of kOnceDone. uint32_t old_control = kOnceInit; - if (control->compare_exchange_strong(old_control, kOnceRunning, - std::memory_order_relaxed) || - base_internal::SpinLockWait(control, static_cast<int>(std::size(trans)), - trans, scheduling_mode) == kOnceInit) { + bool run_fn = control->compare_exchange_strong(old_control, kOnceRunning, + std::memory_order_relaxed); + if (!run_fn) { + const int64_t wait_start_time = base_internal::CycleClock::Now(); + const uint32_t wait_status = base_internal::SpinLockWait( + control, static_cast<int>(std::size(trans)), trans, scheduling_mode); + base_internal::SubmitLockWait(control, wait_start_time, + base_internal::CycleClock::Now()); + run_fn = (wait_status == kOnceInit); + } + if (run_fn) { std::invoke(std::forward<Callable>(fn), std::forward<Args>(args)...); old_control = control->exchange(base_internal::kOnceDone, std::memory_order_release);
diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index d9c9687..19befaa 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc
@@ -84,17 +84,13 @@ // kProfileTimestampShift below) are not reported. static constexpr int64_t kMinProfiledWaitCycles = 1 << 7; -// Report how long an acquiring thread was blocked, measured from the point -// at which it gave up spinning for the lock. -static void ReportLockWait(SpinLock& lock, int64_t wait_start_time, - int64_t wait_end_time) { +void SubmitLockWait(const void* absl_nonnull lock, int64_t wait_start_time, + int64_t wait_end_time) { const int64_t wait_cycles = wait_end_time - wait_start_time; if (wait_cycles < kMinProfiledWaitCycles) { return; } - ABSL_TSAN_MUTEX_PRE_DIVERT(&lock, 0); - submit_lock_profile_data(&lock, wait_cycles); - ABSL_TSAN_MUTEX_POST_DIVERT(&lock, 0); + submit_lock_profile_data(lock, wait_cycles); } // Monitor the lock to see if its value changes within some time period @@ -191,7 +187,9 @@ lock_value = TryLockInternal(lock_value, wait_cycles); } - ReportLockWait(*this, wait_start_time, now); + ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0); + SubmitLockWait(this, wait_start_time, now); + ABSL_TSAN_MUTEX_POST_DIVERT(this, 0); } void SpinLock::SlowUnlock(uint32_t lock_value) {
diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index 82a91ad..da7843b 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h
@@ -294,6 +294,13 @@ // Public interface ends here. //------------------------------------------------------------------------------ +// Reports how long an acquiring thread was blocked over +// `[wait_start_time, wait_end_time]` to the profiler registered with +// RegisterSpinLockLockProfiler(), if any. Callers inside a TSan mutex ignore +// region (such as SpinLock::SlowLock) must divert before calling this function. +void SubmitLockWait(const void* absl_nonnull lock, int64_t wait_start_time, + int64_t wait_end_time); + // If (result & kSpinLockHeld) == 0, then *this was successfully locked. // Otherwise, returns last observed value for lockword_. inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value,
diff --git a/absl/base/spinlock_profiling_test.cc b/absl/base/spinlock_profiling_test.cc index 16c1e2a..48abbe0 100644 --- a/absl/base/spinlock_profiling_test.cc +++ b/absl/base/spinlock_profiling_test.cc
@@ -21,6 +21,7 @@ #include <thread> // NOLINT(build/c++11) #include "gtest/gtest.h" +#include "absl/base/call_once.h" #include "absl/base/config.h" #include "absl/base/internal/cycleclock.h" #include "absl/base/internal/scheduling_mode.h" @@ -134,6 +135,87 @@ EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); } +TEST_F(SpinLockProfiling, CallOnceContentionIsReported) { + once_flag once; + profiled_lock = ControlWord(&once); + + constexpr int64_t kContendedHoldCycles = int64_t{1} << 20; + Notification started; + std::thread initializer([&] { + absl::call_once(once, [&] { + started.Notify(); + while (ControlWord(&once)->load(std::memory_order_relaxed) != + kOnceWaiter) { + std::this_thread::yield(); + } + const int64_t end = CycleClock::Now() + kContendedHoldCycles; + while (CycleClock::Now() < end) { + } + }); + }); + started.WaitForNotification(); + absl::call_once(once, [] {}); + initializer.join(); + profiled_lock = nullptr; + + // Only the blocked waiter reports contention; the slow initializer does not, + // and call_once has no unlock-side contention report. + EXPECT_EQ(1, lock_profiler_calls.load(std::memory_order_relaxed)); + EXPECT_GE(lock_profiler_cycles.load(std::memory_order_relaxed), + kContendedHoldCycles); + EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); +} + +TEST_F(SpinLockProfiling, LowLevelCallOnceContentionIsReported) { + once_flag once; + profiled_lock = ControlWord(&once); + + constexpr int64_t kContendedHoldCycles = int64_t{1} << 20; + std::atomic<bool> started{false}; + std::thread initializer([&] { + LowLevelCallOnce(&once, [&] { + started.store(true, std::memory_order_relaxed); + while (ControlWord(&once)->load(std::memory_order_relaxed) != + kOnceWaiter) { + std::this_thread::yield(); + } + const int64_t end = CycleClock::Now() + kContendedHoldCycles; + while (CycleClock::Now() < end) { + } + }); + }); + while (!started.load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + LowLevelCallOnce(&once, [] {}); + initializer.join(); + profiled_lock = nullptr; + + EXPECT_EQ(1, lock_profiler_calls.load(std::memory_order_relaxed)); + EXPECT_GE(lock_profiler_cycles.load(std::memory_order_relaxed), + kContendedHoldCycles); + EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); +} + +TEST_F(SpinLockProfiling, UncontendedCallOnceIsNotReported) { + once_flag once; + profiled_lock = ControlWord(&once); + + constexpr int64_t kInitCycles = int64_t{1} << 20; + absl::call_once(once, [&] { + const int64_t end = CycleClock::Now() + kInitCycles; + while (CycleClock::Now() < end) { + } + }); + for (int i = 0; i < 1000; ++i) { + absl::call_once(once, [] {}); + } + profiled_lock = nullptr; + + EXPECT_EQ(0, lock_profiler_calls.load(std::memory_order_relaxed)); + EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); +} + } // namespace } // namespace base_internal ABSL_NAMESPACE_END