Add RegisterSpinLockLockProfiler, reporting each acquirer's wait. Since 2022, Mutex has reported both the acquiring thread's callstack and time to acquisition and the releasing thread's callstack when contended. SpinLock retained the prior behavior and its contention reports all go into unlocking delays. This adds the equivalent hook for SpinLock. PiperOrigin-RevId: 994482292
diff --git a/absl/base/BUILD.bazel b/absl/base/BUILD.bazel index b242b90..66f173e 100644 --- a/absl/base/BUILD.bazel +++ b/absl/base/BUILD.bazel
@@ -581,6 +581,25 @@ ], ) +cc_test( + name = "spinlock_profiling_test", + size = "small", + srcs = ["spinlock_profiling_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", # b/122473323 + ], + deps = [ + ":base", + ":base_internal", + ":config", + "//absl/synchronization", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + cc_library( name = "spinlock_benchmark_common", testonly = True,
diff --git a/absl/base/CMakeLists.txt b/absl/base/CMakeLists.txt index d1ce0f9..dc72d45 100644 --- a/absl/base/CMakeLists.txt +++ b/absl/base/CMakeLists.txt
@@ -538,6 +538,21 @@ 4 ) +absl_cc_test( + NAME + spinlock_profiling_test + SRCS + "spinlock_profiling_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::synchronization + GTest::gtest_main +) + # Internal-only target, do not depend on directly. absl_cc_library( NAME
diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 9046a00..d9c9687 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc
@@ -28,6 +28,7 @@ #include "absl/base/internal/spinlock_wait.h" #include "absl/base/internal/sysinfo.h" /* For NumCPUs() */ #include "absl/base/internal/tsan_mutex_interface.h" +#include "absl/base/nullability.h" // Description of lock-word: // 31..00: [............................3][2][1][0] @@ -65,11 +66,37 @@ const void* lock, int64_t wait_cycles)> submit_unlock_profile_data; -void RegisterSpinLockProfiler(void (*fn)(const void* contendedlock, - int64_t wait_cycles)) { +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static AtomicHook<void (*)( + const void* lock, int64_t wait_cycles)> + submit_lock_profile_data; + +void RegisterSpinLockProfiler(void (*absl_nonnull fn)( + const void* absl_nonnull contendedlock, int64_t wait_cycles)) { submit_unlock_profile_data.Store(fn); } +void RegisterSpinLockLockProfiler(void (*absl_nonnull fn)( + const void* absl_nonnull contendedlock, int64_t wait_cycles)) { + submit_lock_profile_data.Store(fn); +} + +// Waits shorter than one profiling timestamp unit (see +// kProfileTimestampShift below) are not reported. +static constexpr int64_t kMinProfiledWaitCycles = 1 << 7; + +// Report how long an acquiring thread was blocked, measured from the point +// at which it gave up spinning for the lock. +static void ReportLockWait(SpinLock& lock, int64_t wait_start_time, + int64_t wait_end_time) { + const int64_t wait_cycles = wait_end_time - wait_start_time; + if (wait_cycles < kMinProfiledWaitCycles) { + return; + } + ABSL_TSAN_MUTEX_PRE_DIVERT(&lock, 0); + submit_lock_profile_data(&lock, wait_cycles); + ABSL_TSAN_MUTEX_POST_DIVERT(&lock, 0); +} + // Monitor the lock to see if its value changes within some time period // (adaptive_spin_count_ loop iterations). The last value read from the lock // is returned from the method. @@ -111,9 +138,10 @@ // The lock was not obtained initially, so this thread needs to wait for // it. Record the current timestamp in the local variable wait_start_time - // so the total wait time can be stored in the lockword once this thread - // obtains the lock. - int64_t wait_start_time = CycleClock::Now(); + // so the total wait time can be reported once this thread obtains the lock, + // and stored in the lockword for the unlock-side profiler. + const int64_t wait_start_time = CycleClock::Now(); + int64_t now = wait_start_time; uint32_t wait_cycles = 0; int lock_wait_call_count = 0; while ((lock_value & kSpinLockHeld) != 0) { @@ -158,9 +186,12 @@ // Spin again after returning from the wait routine to give this thread // some chance of obtaining the lock. lock_value = SpinLoop(); - wait_cycles = EncodeWaitCycles(wait_start_time, CycleClock::Now()); + now = CycleClock::Now(); + wait_cycles = EncodeWaitCycles(wait_start_time, now); lock_value = TryLockInternal(lock_value, wait_cycles); } + + ReportLockWait(*this, wait_start_time, now); } void SpinLock::SlowUnlock(uint32_t lock_value) {
diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index 93a9f1c..82a91ad 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h
@@ -42,6 +42,7 @@ #include "absl/base/internal/scheduling_mode.h" #include "absl/base/internal/tsan_mutex_interface.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/base/thread_annotations.h" namespace tcmalloc { @@ -174,6 +175,7 @@ // Provide access to protected method above. Use for testing only. friend struct SpinLockTest; + friend struct SpinLockProfilingTestPeer; friend class tcmalloc::tcmalloc_internal::AllocationGuardSpinLockHolder; friend class tcmalloc::tcmalloc_internal::Static; @@ -256,21 +258,37 @@ ABSL_EXCLUSIVE_LOCK_FUNCTION(l) : std::lock_guard<SpinLock>(l) {} ABSL_DEPRECATE_AND_INLINE() - inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) + inline explicit SpinLockHolder(SpinLock* absl_nonnull l) + ABSL_EXCLUSIVE_LOCK_FUNCTION(l) : SpinLockHolder(*l) {} inline ~SpinLockHolder() ABSL_UNLOCK_FUNCTION() = default; }; -// Register a hook for profiling support. +// Register hooks for profiling support. // -// The function pointer registered here will be called whenever a spinlock is -// contended. The callback is given an opaque handle to the contended spinlock -// and the number of wait cycles. This is thread-safe, but only a single -// profiler can be registered. It is an error to call this function multiple +// The callbacks are given an opaque handle to the contended spinlock and a +// number of wait cycles as measured by +// //absl/base/internal/cycleclock.h, which may not be real "cycle" +// counts. Registration is thread-safe, but only a single profiler can be +// registered for each hook. It is an error to call these functions multiple // times with different arguments. -void RegisterSpinLockProfiler(void (*fn)(const void* lock, - int64_t wait_cycles)); +// +// The function registered with RegisterSpinLockProfiler() is called when a +// thread releases a spinlock whose acquisition was contended. `wait_cycles` is +// the time the releasing thread spent waiting to acquire the lock, as stored +// in the lock word by its acquisition. +void RegisterSpinLockProfiler(void (*absl_nonnull fn)( + const void* absl_nonnull lock, int64_t wait_cycles)); + +// The function registered with RegisterSpinLockLockProfiler() is called when a +// thread acquires a spinlock that it could not acquire by spinning. +// `wait_cycles` is the time for which the acquiring thread was blocked, +// measured from the point at which it gave up spinning. Acquisitions that +// spinning resolves are not reported, so this undercounts the time spent +// waiting for spinlocks by at most one spin per acquisition. +void RegisterSpinLockLockProfiler(void (*absl_nonnull fn)( + const void* absl_nonnull lock, int64_t wait_cycles)); //------------------------------------------------------------------------------ // Public interface ends here.
diff --git a/absl/base/spinlock_profiling_test.cc b/absl/base/spinlock_profiling_test.cc new file mode 100644 index 0000000..16c1e2a --- /dev/null +++ b/absl/base/spinlock_profiling_test.cc
@@ -0,0 +1,140 @@ +// Copyright 2026 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Exercises the SpinLock contention profiling hooks. The hooks can only be +// registered once per process, so this test lives in its own binary and must +// not be linked together with another profiler. + +#include <atomic> +#include <cstdint> +#include <thread> // NOLINT(build/c++11) + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/spinlock.h" +#include "absl/synchronization/notification.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +struct SpinLockProfilingTestPeer { + // True once a waiter has marked itself as about to sleep in the lock word. + static bool HasSleeper(const SpinLock& l) { + return (l.lockword_.load(std::memory_order_relaxed) & + SpinLock::kSpinLockSleeper) != 0; + } +}; + +namespace { + +const void* profiled_lock = nullptr; +std::atomic<int64_t> lock_profiler_calls{0}; +std::atomic<int64_t> lock_profiler_cycles{0}; +std::atomic<int64_t> unlock_profiler_calls{0}; +std::atomic<int64_t> unlock_profiler_cycles{0}; +// Number of times the unlock profiler ran for an acquisition whose lock +// profiler had not yet run. +std::atomic<int64_t> unlock_before_lock_calls{0}; + +void LockProfiler(const void* lock, int64_t wait_cycles) { + if (lock != profiled_lock) return; + lock_profiler_calls.fetch_add(1, std::memory_order_relaxed); + lock_profiler_cycles.fetch_add(wait_cycles, std::memory_order_relaxed); +} + +void UnlockProfiler(const void* lock, int64_t wait_cycles) { + if (lock != profiled_lock) return; + if (lock_profiler_calls.load(std::memory_order_relaxed) == 0) { + unlock_before_lock_calls.fetch_add(1, std::memory_order_relaxed); + } + unlock_profiler_calls.fetch_add(1, std::memory_order_relaxed); + unlock_profiler_cycles.fetch_add(wait_cycles, std::memory_order_relaxed); +} + +class SpinLockProfiling : public ::testing::Test { + protected: + void SetUp() override { + RegisterSpinLockProfiler(&UnlockProfiler); + RegisterSpinLockLockProfiler(&LockProfiler); + lock_profiler_calls.store(0, std::memory_order_relaxed); + lock_profiler_cycles.store(0, std::memory_order_relaxed); + unlock_profiler_calls.store(0, std::memory_order_relaxed); + unlock_profiler_cycles.store(0, std::memory_order_relaxed); + unlock_before_lock_calls.store(0, std::memory_order_relaxed); + } + + void TearDown() override { profiled_lock = nullptr; } +}; + +TEST_F(SpinLockProfiling, ContentionProfilers) { + SpinLock spinlock(SCHEDULE_KERNEL_ONLY); + profiled_lock = &spinlock; + + // Hold the lock until a waiter is about to sleep on it, then keep holding + // it for kContendedHoldCycles so that the waiter's wait is comfortably above + // the reporting threshold. + constexpr int64_t kContendedHoldCycles = int64_t{1} << 20; + Notification held; + std::thread holder([&] { + spinlock.lock(); + held.Notify(); + while (!SpinLockProfilingTestPeer::HasSleeper(spinlock)) { + std::this_thread::yield(); + } + const int64_t end = CycleClock::Now() + kContendedHoldCycles; + while (CycleClock::Now() < end) { + } + spinlock.unlock(); + }); + held.WaitForNotification(); + // Causes contention, since `holder` already holds the lock. + spinlock.lock(); + spinlock.unlock(); + holder.join(); + profiled_lock = nullptr; + + // The waiter's acquisition reported its own wait. + EXPECT_EQ(1, lock_profiler_calls.load(std::memory_order_relaxed)); + EXPECT_GE(lock_profiler_cycles.load(std::memory_order_relaxed), + kContendedHoldCycles); + // The waiter's release reported the wait stored in the lock word by its + // acquisition. The holder's acquisition was uncontended. + EXPECT_EQ(1, unlock_profiler_calls.load(std::memory_order_relaxed)); + EXPECT_GE(unlock_profiler_cycles.load(std::memory_order_relaxed), + kContendedHoldCycles); + // An acquisition is reported by the lock profiler before its release is + // reported by the unlock profiler. + EXPECT_EQ(0, unlock_before_lock_calls.load(std::memory_order_relaxed)); +} + +TEST_F(SpinLockProfiling, UncontendedAcquisitionsAreNotReported) { + SpinLock spinlock(SCHEDULE_KERNEL_ONLY); + profiled_lock = &spinlock; + for (int i = 0; i < 1000; ++i) { + spinlock.lock(); + spinlock.unlock(); + } + profiled_lock = nullptr; + + EXPECT_EQ(0, lock_profiler_calls.load(std::memory_order_relaxed)); + EXPECT_EQ(0, unlock_profiler_calls.load(std::memory_order_relaxed)); +} + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl