Use non-stack storage for stack trace buffers Doing this opportunistically allows us to avoid performance overhead in the vast majority of calls (rare, non-reentrant ones) while simultaneously minimizing stack space usage. PiperOrigin-RevId: 831560881 Change-Id: Idc6ba1dd0dcf1b4aaf3ee7cf468054bcfdcf90af
diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index d490e65..c49d621 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake
@@ -128,6 +128,8 @@ "debugging/internal/address_is_readable.h" "debugging/internal/addresses.h" "debugging/internal/bounded_utf8_length_sequence.h" + "debugging/internal/borrowed_fixup_buffer.h" + "debugging/internal/borrowed_fixup_buffer.cc" "debugging/internal/decode_rust_punycode.cc" "debugging/internal/decode_rust_punycode.h" "debugging/internal/demangle.cc"
diff --git a/absl/debugging/BUILD.bazel b/absl/debugging/BUILD.bazel index 7cc053e..aad5e28 100644 --- a/absl/debugging/BUILD.bazel +++ b/absl/debugging/BUILD.bazel
@@ -36,6 +36,33 @@ licenses(["notice"]) cc_library( + name = "borrowed_fixup_buffer", + srcs = ["internal/borrowed_fixup_buffer.cc"], + hdrs = ["internal/borrowed_fixup_buffer.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:malloc_internal", + "//absl/hash", + ], +) + +cc_test( + name = "borrowed_fixup_buffer_test", + srcs = ["internal/borrowed_fixup_buffer_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":borrowed_fixup_buffer", + "//absl/base:config", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( name = "stacktrace", srcs = [ "internal/stacktrace_aarch64-inl.inc", @@ -54,6 +81,7 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":borrowed_fixup_buffer", ":debugging_internal", "//absl/base:config", "//absl/base:core_headers", @@ -69,6 +97,7 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":borrowed_fixup_buffer", ":stacktrace", "//absl/base:config", "//absl/base:core_headers", @@ -448,6 +477,7 @@ ":stacktrace", "//absl/base:config", "//absl/base:core_headers", + "//absl/cleanup", "@google_benchmark//:benchmark_main", ], )
diff --git a/absl/debugging/CMakeLists.txt b/absl/debugging/CMakeLists.txt index d8249fe..ab3a795 100644 --- a/absl/debugging/CMakeLists.txt +++ b/absl/debugging/CMakeLists.txt
@@ -18,6 +18,38 @@ absl_cc_library( NAME + borrowed_fixup_buffer + SRCS + "internal/borrowed_fixup_buffer.cc" + HDRS + "internal/borrowed_fixup_buffer.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::hash + absl::malloc_internal + PUBLIC +) + +absl_cc_test( + NAME + borrowed_fixup_buffer_test + SRCS + "internal/borrowed_fixup_buffer_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::borrowed_fixup_buffer + absl::config + GTest::gmock_main +) + +absl_cc_library( + NAME stacktrace HDRS "stacktrace.h" @@ -38,6 +70,7 @@ LINKOPTS $<$<BOOL:${EXECINFO_LIBRARY}>:${EXECINFO_LIBRARY}> DEPS + absl::borrowed_fixup_buffer absl::debugging_internal absl::config absl::core_headers
diff --git a/absl/debugging/internal/borrowed_fixup_buffer.cc b/absl/debugging/internal/borrowed_fixup_buffer.cc new file mode 100644 index 0000000..dae78a7 --- /dev/null +++ b/absl/debugging/internal/borrowed_fixup_buffer.cc
@@ -0,0 +1,118 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/internal/borrowed_fixup_buffer.h" + +#include <assert.h> +#include <limits.h> +#include <stddef.h> +#include <stdint.h> + +#include <atomic> +#include <iterator> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/low_level_alloc.h" +#include "absl/hash/hash.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace internal_stacktrace { + +// A buffer for holding fix-up information for stack traces of common sizes_. +struct BorrowedFixupBuffer::FixupStackBuffer { + static constexpr size_t kMaxStackElements = 128; // Can be reduced if needed + std::atomic_flag in_use{}; + uintptr_t frames[kMaxStackElements]; + int sizes[kMaxStackElements]; + + ABSL_CONST_INIT static FixupStackBuffer g_instances[kNumStaticBuffers]; +}; + +ABSL_CONST_INIT BorrowedFixupBuffer::FixupStackBuffer + BorrowedFixupBuffer::FixupStackBuffer::g_instances[kNumStaticBuffers] = {}; + +BorrowedFixupBuffer::~BorrowedFixupBuffer() { + if (borrowed_) { + Unlock(); + } else { + base_internal::LowLevelAlloc::Free(frames_); + } +} + +BorrowedFixupBuffer::BorrowedFixupBuffer(size_t length) { + FixupStackBuffer* fixup_buffer = + 0 < length && length <= FixupStackBuffer::kMaxStackElements ? TryLock() + : nullptr; + borrowed_ = fixup_buffer != nullptr; + if (borrowed_) { + InitViaBorrow(fixup_buffer); + } else { + InitViaAllocation(length); + } +} + +void BorrowedFixupBuffer::InitViaBorrow(FixupStackBuffer* borrowed_buffer) { + assert(borrowed_); + frames_ = borrowed_buffer->frames; + sizes_ = borrowed_buffer->sizes; +} + +void BorrowedFixupBuffer::InitViaAllocation(size_t length) { + static_assert(alignof(decltype(*frames_)) >= alignof(decltype(*sizes_)), + "contiguous layout assumes decreasing alignment, otherwise " + "padding may be needed in the middle"); + assert(!borrowed_); + + base_internal::InitSigSafeArena(); + void* buf = base_internal::LowLevelAlloc::AllocWithArena( + length * (sizeof(*frames_) + sizeof(*sizes_)), + base_internal::SigSafeArena()); + + if (buf == nullptr) { + frames_ = nullptr; + sizes_ = nullptr; + return; + } + + frames_ = new (buf) uintptr_t[length]; + sizes_ = new (static_cast<void*>(static_cast<unsigned char*>(buf) + + length * sizeof(*frames_))) int[length]; +} + +BorrowedFixupBuffer::FixupStackBuffer* BorrowedFixupBuffer::Find() { + size_t i = absl::Hash<const void*>()(this) % + std::size(FixupStackBuffer::g_instances); + return &FixupStackBuffer::g_instances[i]; +} + +[[nodiscard]] BorrowedFixupBuffer::FixupStackBuffer* +BorrowedFixupBuffer::TryLock() { + FixupStackBuffer* instance = Find(); + // Use memory_order_acquire to ensure that no reads and writes on the borrowed + // buffer are reordered before the borrowing. + return !instance->in_use.test_and_set(std::memory_order_acquire) ? instance + : nullptr; +} + +void BorrowedFixupBuffer::Unlock() { + // Use memory_order_release to ensure that no reads and writes on the borrowed + // buffer are reordered after the borrowing. + Find()->in_use.clear(std::memory_order_release); +} + +} // namespace internal_stacktrace +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/debugging/internal/borrowed_fixup_buffer.h b/absl/debugging/internal/borrowed_fixup_buffer.h new file mode 100644 index 0000000..c5ea7a3 --- /dev/null +++ b/absl/debugging/internal/borrowed_fixup_buffer.h
@@ -0,0 +1,74 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_DEBUGGING_INTERNAL_BORROWED_FIXUP_BUFFER_H_ +#define ABSL_DEBUGGING_INTERNAL_BORROWED_FIXUP_BUFFER_H_ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace internal_stacktrace { + +// An RAII type that temporarily acquires a buffer for stack trace fix-ups from +// a pool of preallocated buffers, or attempts to allocate a new buffer if no +// such buffer is available. +// When destroyed, returns the buffer to the pool if it borrowed successfully, +// otherwise deallocates any previously allocated buffer. +class BorrowedFixupBuffer { + public: + static constexpr size_t kNumStaticBuffers = 64; + ~BorrowedFixupBuffer(); + + // The number of frames to allocate space for. Note that allocations can fail. + explicit BorrowedFixupBuffer(size_t length); + + uintptr_t* frames() const { return frames_; } + int* sizes() const { return sizes_; } + + private: + uintptr_t* frames_; + int* sizes_; + + // Have we borrowed a pre-existing buffer (vs. allocated our own)? + bool borrowed_; + + struct FixupStackBuffer; + + void InitViaBorrow(FixupStackBuffer* borrowed_buffer); + void InitViaAllocation(size_t length); + + // Returns a non-null pointer to a buffer that could be potentially borrowed. + FixupStackBuffer* Find(); + + // Attempts to opportunistically borrow a small buffer in a thread- and + // signal-safe manner. Returns nullptr on failure. + [[nodiscard]] FixupStackBuffer* TryLock(); + + // Returns the borrowed buffer. + void Unlock(); + + BorrowedFixupBuffer(const BorrowedFixupBuffer&) = delete; + BorrowedFixupBuffer& operator=(const BorrowedFixupBuffer&) = delete; +}; + +} // namespace internal_stacktrace +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_BORROWED_FIXUP_BUFFER_H_
diff --git a/absl/debugging/internal/borrowed_fixup_buffer_test.cc b/absl/debugging/internal/borrowed_fixup_buffer_test.cc new file mode 100644 index 0000000..a856c5d --- /dev/null +++ b/absl/debugging/internal/borrowed_fixup_buffer_test.cc
@@ -0,0 +1,97 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/internal/borrowed_fixup_buffer.h" + +#include <stddef.h> +#include <stdint.h> + +#include <algorithm> +#include <functional> +#include <memory> + +#include "gtest/gtest.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace internal_stacktrace { +namespace { + +TEST(BorrowedFixupBuffer, ProperReuse) { + uintptr_t first_borrowed_frame = 0; + uintptr_t first_borrowed_size = 0; + + // Ensure that we borrow the same buffer each time, indicating proper reuse. + // Disable loop unrolling. We need all iterations to match exactly, to coax + // reuse of the the same underlying buffer. +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC unroll 1 // <= 1 disables unrolling +#endif + for (int i = 0; i < 100; ++i) { + BorrowedFixupBuffer buf0(0); + EXPECT_EQ(buf0.frames(), nullptr); + EXPECT_EQ(buf0.sizes(), nullptr); + + BorrowedFixupBuffer buf1(1); + EXPECT_NE(buf1.frames(), nullptr); + EXPECT_NE(buf1.sizes(), nullptr); + if (first_borrowed_frame == 0) { + first_borrowed_frame = reinterpret_cast<uintptr_t>(buf1.frames()); + } else { + EXPECT_EQ(reinterpret_cast<uintptr_t>(buf1.frames()), + first_borrowed_frame); + } + if (first_borrowed_size == 0) { + first_borrowed_size = reinterpret_cast<uintptr_t>(buf1.sizes()); + } else { + EXPECT_EQ(reinterpret_cast<uintptr_t>(buf1.sizes()), first_borrowed_size); + } + + BorrowedFixupBuffer buf2(2); + EXPECT_NE(buf2.frames(), buf1.frames()); + EXPECT_NE(buf2.sizes(), buf1.sizes()); + EXPECT_NE(buf2.frames(), nullptr); + EXPECT_NE(buf2.sizes(), nullptr); + } +} + +TEST(BorrowedFixupBuffer, NoOverlap) { + using BufferPtr = std::unique_ptr<BorrowedFixupBuffer>; + static constexpr std::less<const void*> less; + static constexpr size_t kBufLen = 5; + static constexpr size_t kNumBuffers = + BorrowedFixupBuffer::kNumStaticBuffers * 37 + 1; + + auto bufs = std::make_unique<BufferPtr[]>(kNumBuffers); + for (size_t i = 0; i < kNumBuffers; ++i) { + bufs[i] = std::make_unique<BorrowedFixupBuffer>(kBufLen); + } + + std::sort(bufs.get(), bufs.get() + kNumBuffers, + [](const BufferPtr& a, const BufferPtr& b) { + return less(a->frames(), b->frames()); + }); + + // Verify there are no overlaps + for (size_t i = 1; i < kNumBuffers; ++i) { + EXPECT_FALSE(less(bufs[i]->frames(), bufs[i - 1]->frames() + kBufLen)); + EXPECT_FALSE(less(bufs[i]->sizes(), bufs[i - 1]->sizes() + kBufLen)); + } +} + +} // namespace +} // namespace internal_stacktrace +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/debugging/stacktrace.cc b/absl/debugging/stacktrace.cc index acc8b66..aee065d 100644 --- a/absl/debugging/stacktrace.cc +++ b/absl/debugging/stacktrace.cc
@@ -42,14 +42,12 @@ #include <algorithm> #include <atomic> -#include <iterator> -#include <type_traits> #include "absl/base/attributes.h" #include "absl/base/config.h" -#include "absl/base/internal/low_level_alloc.h" #include "absl/base/optimization.h" #include "absl/base/port.h" +#include "absl/debugging/internal/borrowed_fixup_buffer.h" #include "absl/debugging/internal/stacktrace_config.h" #if defined(ABSL_STACKTRACE_INL_HEADER) @@ -75,37 +73,14 @@ typedef int (*Unwinder)(void**, int*, int, int, const void*, int*); std::atomic<Unwinder> custom; -constexpr size_t kMinPageSize = 4096; - -struct FixupBuffer { - static constexpr size_t kMaxStackElements = 128; // Can be reduced if needed - uintptr_t frames[kMaxStackElements]; - int sizes[kMaxStackElements]; -}; -static_assert(std::is_trivially_default_constructible_v<FixupBuffer>); -static_assert(sizeof(FixupBuffer) < kMinPageSize / 2, - "buffer size should no larger than a small fraction of a page, " - "to avoid stack overflows"); - template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> -ABSL_ATTRIBUTE_ALWAYS_INLINE inline int Unwind( - void** result, uintptr_t* frames, int* sizes, size_t max_depth, - int skip_count, const void* uc, int* min_dropped_frames, - FixupBuffer* fixup_buffer /* if NULL, fixups are skipped */) { - // Allocate a buffer dynamically, using the signal-safe allocator. - static constexpr auto allocate = [](size_t num_bytes) -> void* { - base_internal::InitSigSafeArena(); - return base_internal::LowLevelAlloc::AllocWithArena( - num_bytes, base_internal::SigSafeArena()); - }; - - // We only need to free the buffers if we allocated them with the signal-safe - // allocator. - bool must_free_frames = false; - bool must_free_sizes = false; - - bool unwind_with_fixup = - fixup_buffer != nullptr && internal_stacktrace::ShouldFixUpStack(); +ABSL_ATTRIBUTE_ALWAYS_INLINE inline int Unwind(void** result, uintptr_t* frames, + int* sizes, size_t max_depth, + int skip_count, const void* uc, + int* min_dropped_frames, + bool unwind_with_fixup = true) { + unwind_with_fixup = + unwind_with_fixup && internal_stacktrace::ShouldFixUpStack(); #ifdef _WIN32 if (unwind_with_fixup) { @@ -117,29 +92,17 @@ } #endif - if (unwind_with_fixup) { - // Some implementations of FixUpStack may need to be passed frame - // information from Unwind, even if the caller doesn't need that - // information. We allocate the necessary buffers for such implementations - // here. - - if (frames == nullptr) { - if (max_depth <= std::size(fixup_buffer->frames)) { - frames = fixup_buffer->frames; - } else { - frames = static_cast<uintptr_t*>(allocate(max_depth * sizeof(*frames))); - must_free_frames = true; - } - } - - if (sizes == nullptr) { - if (max_depth <= std::size(fixup_buffer->sizes)) { - sizes = fixup_buffer->sizes; - } else { - sizes = static_cast<int*>(allocate(max_depth * sizeof(*sizes))); - must_free_sizes = true; - } - } + // Some implementations of FixUpStack may need to be passed frame + // information from Unwind, even if the caller doesn't need that + // information. We allocate the necessary buffers for such implementations + // here. + const internal_stacktrace::BorrowedFixupBuffer fixup_buffer( + unwind_with_fixup ? max_depth : 0); + if (frames == nullptr) { + frames = fixup_buffer.frames(); + } + if (sizes == nullptr) { + sizes = fixup_buffer.sizes(); } Unwinder g = custom.load(std::memory_order_acquire); @@ -167,14 +130,6 @@ internal_stacktrace::FixUpStack(result, frames, sizes, max_depth, size); } - if (must_free_sizes) { - base_internal::LowLevelAlloc::Free(sizes); - } - - if (must_free_frames) { - base_internal::LowLevelAlloc::Free(frames); - } - ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); return static_cast<int>(size); } @@ -184,10 +139,9 @@ ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int internal_stacktrace::GetStackFrames(void** result, uintptr_t* frames, int* sizes, int max_depth, int skip_count) { - FixupBuffer fixup_stack_buf; return Unwind<true, false>(result, frames, sizes, static_cast<size_t>(max_depth), skip_count, - nullptr, nullptr, &fixup_stack_buf); + nullptr, nullptr); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int @@ -195,10 +149,9 @@ int* sizes, int max_depth, int skip_count, const void* uc, int* min_dropped_frames) { - FixupBuffer fixup_stack_buf; return Unwind<true, true>(result, frames, sizes, static_cast<size_t>(max_depth), skip_count, uc, - min_dropped_frames, &fixup_stack_buf); + min_dropped_frames); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int @@ -206,24 +159,22 @@ int skip_count) { return Unwind<false, false>(result, nullptr, nullptr, static_cast<size_t>(max_depth), skip_count, - nullptr, nullptr, nullptr); + nullptr, nullptr, /*unwind_with_fixup=*/false); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackTrace( void** result, int max_depth, int skip_count) { - FixupBuffer fixup_stack_buf; return Unwind<false, false>(result, nullptr, nullptr, static_cast<size_t>(max_depth), skip_count, - nullptr, nullptr, &fixup_stack_buf); + nullptr, nullptr); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackTraceWithContext(void** result, int max_depth, int skip_count, const void* uc, int* min_dropped_frames) { - FixupBuffer fixup_stack_buf; return Unwind<false, true>(result, nullptr, nullptr, static_cast<size_t>(max_depth), skip_count, uc, - min_dropped_frames, &fixup_stack_buf); + min_dropped_frames); } void SetStackUnwinder(Unwinder w) {
diff --git a/absl/debugging/stacktrace_benchmark.cc b/absl/debugging/stacktrace_benchmark.cc index 9360baf..eef9850 100644 --- a/absl/debugging/stacktrace_benchmark.cc +++ b/absl/debugging/stacktrace_benchmark.cc
@@ -12,12 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <stddef.h> +#include <stdint.h> + #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" #include "absl/debugging/stacktrace.h" #include "benchmark/benchmark.h" +static bool g_enable_fixup = false; + +#if ABSL_HAVE_ATTRIBUTE_WEAK +// Override these weak symbols if possible. +bool absl::internal_stacktrace::ShouldFixUpStack() { return g_enable_fixup; } +void absl::internal_stacktrace::FixUpStack(void**, uintptr_t*, int*, size_t, + size_t&) {} +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace { @@ -42,14 +55,24 @@ func(state, --x, depth); } +template <bool EnableFixup> void BM_GetStackTrace(benchmark::State& state) { + const Cleanup restore_state( + [prev = g_enable_fixup]() { g_enable_fixup = prev; }); + g_enable_fixup = EnableFixup; int depth = state.range(0); for (auto s : state) { func(state, depth, depth); } } -BENCHMARK(BM_GetStackTrace)->DenseRange(10, kMaxStackDepth, 10); +#if ABSL_HAVE_ATTRIBUTE_WEAK +auto& BM_GetStackTraceWithFixup = BM_GetStackTrace<true>; +BENCHMARK(BM_GetStackTraceWithFixup)->DenseRange(10, kMaxStackDepth, 10); +#endif + +auto& BM_GetStackTraceWithoutFixup = BM_GetStackTrace<false>; +BENCHMARK(BM_GetStackTraceWithoutFixup)->DenseRange(10, kMaxStackDepth, 10); } // namespace ABSL_NAMESPACE_END } // namespace absl