Copy AnyInvocable trivial storage in pointer-sized words and skip copies on dispose to avoid store-forwarding stalls TypeErasedState spans two pointer-sized words (16 bytes on 64-bit platforms, aligned to 16 bytes on 64-bit SysV/Darwin), so `*to = *from` in LocalManagerTrivial() and `to->remote = from->remote` in RemoteManagerTrivial() compile to a 2-word SIMD load/store (such as `vmovaps (%rsi), %xmm0` / `vmovaps %xmm0, (%rdx)` on x86-64 and `ldr q0, [x1]` / `str q0, [x2]` on AArch64). When an absl::AnyInvocable is constructed from a function pointer, single-pointer lambda, sub-word scalar capture, or two-word remote state and then moved or destroyed (LocalManagerTrivial() previously self-copied on FunctionToCall::dispose as well), the 2-word SIMD load spans narrower scalar stores—or a 2-word SIMD store precedes scalar reads in LocalInvoker/RemoteInvoker—causing store-to-load forwarding stalls. Avoid 2-word SIMD copies and unnecessary teardown copies using standard C++ without changing TypeErasedState's triviality or implicit-lifetime properties: - Return early on FunctionToCall::dispose in LocalManagerTrivial() and LocalManagerTrivialSingleWord() (matching RemoteManagerTrivial()) to eliminate teardown loads/stores. - Route empty trivially copyable local callables to EmptyManager() (starting `to->storage` lifetime per P0593 as a no-op), single-word (`<= sizeof(void*)`) callables to LocalManagerTrivialSingleWord() (single pointer-sized std::memmove), and multi-word callables to LocalManagerTrivial() (two sequential pointer-sized std::memmove calls). - Copy `remote.target` and `remote.size` field-by-field in RemoteManagerTrivial() instead of via whole-struct assignment. - Stage non-empty trivially copyable local callables through a zeroed word-multiple `unsigned char` buffer into `state_.storage` via pointer-sized std::memcpy calls in InitializeStorage() so narrow sub-word stores do not stall subsequent word-sized loads. - Guard TypeErasedState triviality with static_assert and add unit tests covering empty, sub-word, partial/full multi-word, padded, mutable, move-only, and remote trivial callables. PiperOrigin-RevId: 1019520730
diff --git a/absl/functional/any_invocable_test.h b/absl/functional/any_invocable_test.h index 6d09c12..5c0a36c 100644 --- a/absl/functional/any_invocable_test.h +++ b/absl/functional/any_invocable_test.h
@@ -44,6 +44,12 @@ static_assert(absl::internal_any_invocable::kStorageSize >= sizeof(void*), "These tests assume that the small object storage is at least " "the size of a pointer."); +static_assert( + std::is_trivially_copyable_v<absl::internal_any_invocable::TypeErasedState>, + "TypeErasedState must remain trivially copyable."); +static_assert(std::is_trivially_default_constructible_v< + absl::internal_any_invocable::TypeErasedState>, + "TypeErasedState must remain trivially default-constructible."); namespace absl_any_invocable_test {
diff --git a/absl/functional/any_invocable_test_inst4.cc b/absl/functional/any_invocable_test_inst4.cc index ac21ca9..f5fbcf6 100644 --- a/absl/functional/any_invocable_test_inst4.cc +++ b/absl/functional/any_invocable_test_inst4.cc
@@ -18,10 +18,303 @@ // SKIP_ABSL_INLINE_NAMESPACE_CHECK +#include <cstddef> +#include <cstdint> +#include <type_traits> +#include <utility> + #include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/functional/any_invocable.h" #include "absl/functional/any_invocable_test.h" namespace absl_any_invocable_test { +namespace { + +template <size_t N> +struct ByteArrayFunctor { + unsigned char bytes[N]; + + int operator()(int seed) const noexcept { + int sum = seed; + for (size_t i = 0; i < N; ++i) { + sum += static_cast<int>(bytes[i]) * static_cast<int>(i + 1); + } + return sum; + } +}; + +template <size_t N> +void VerifyByteArrayFunctor() { + static_assert(std::is_trivially_copyable_v<ByteArrayFunctor<N>>); + if constexpr (N <= absl::internal_any_invocable::kStorageSize) { + static_assert( + absl::internal_any_invocable::IsStoredLocally<ByteArrayFunctor<N>>()); + } + + ByteArrayFunctor<N> f1{}; + ByteArrayFunctor<N> f2{}; + for (size_t i = 0; i < N; ++i) { + f1.bytes[i] = static_cast<unsigned char>(i + 3); + f2.bytes[i] = static_cast<unsigned char>(i + 17); + } + const int expected1 = f1(5); + const int expected2 = f2(5); + + absl::AnyInvocable<int(int) const> a(f1); + EXPECT_EQ(a(5), expected1); + + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(5), expected1); + + absl::AnyInvocable<int(int) const> c(std::in_place_type<ByteArrayFunctor<N>>, + f2); + EXPECT_EQ(c(5), expected2); + + b.swap(c); + EXPECT_EQ(b(5), expected2); + EXPECT_EQ(c(5), expected1); + + b = std::move(c); + EXPECT_EQ(b(5), expected1); + b = nullptr; + EXPECT_EQ(b, nullptr); +} + +TEST(AnyInvTestTrivialStorage, SubWordAndMultiWordSizes) { + VerifyByteArrayFunctor<1>(); + VerifyByteArrayFunctor<2>(); + VerifyByteArrayFunctor<3>(); + VerifyByteArrayFunctor<4>(); + VerifyByteArrayFunctor<5>(); + VerifyByteArrayFunctor<6>(); + VerifyByteArrayFunctor<7>(); + VerifyByteArrayFunctor<8>(); + VerifyByteArrayFunctor<9>(); + VerifyByteArrayFunctor<12>(); + VerifyByteArrayFunctor<15>(); + VerifyByteArrayFunctor<16>(); + // Also exercise remote trivially destructible storage + // (`RemoteManagerTrivial`) with default alignment (alignof == 1 <= + // ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT). + VerifyByteArrayFunctor<24>(); +} + +struct CharIntPaddedFunctor { + char c; + int x; + + int operator()(int delta) const noexcept { + return static_cast<int>(c) + x + delta; + } +}; + +struct CharPtrPaddedFunctor { + char c; + const int* p; + + int operator()(int delta) const noexcept { + return static_cast<int>(c) + *p + delta; + } +}; + +struct PtrCharPaddedFunctor { + const int* p; + char c; + + int operator()(int delta) const noexcept { + return static_cast<int>(c) + *p + delta; + } +}; + +// Construct padded structs field-by-field in noinline helpers so padding bytes +// remain uninitialized under sanitizers such as MSan. +ABSL_ATTRIBUTE_NOINLINE CharIntPaddedFunctor MakeCharIntPadded(char c, int x) { + CharIntPaddedFunctor f; + f.c = c; + f.x = x; + return f; +} + +ABSL_ATTRIBUTE_NOINLINE CharPtrPaddedFunctor MakeCharPtrPadded(char c, + const int* p) { + CharPtrPaddedFunctor f; + f.c = c; + f.p = p; + return f; +} + +ABSL_ATTRIBUTE_NOINLINE PtrCharPaddedFunctor MakePtrCharPadded(const int* p, + char c) { + PtrCharPaddedFunctor f; + f.p = p; + f.c = c; + return f; +} + +TEST(AnyInvTestTrivialStorage, InternalAndTailPadding) { + static_assert(std::is_trivially_copyable_v<CharIntPaddedFunctor>); + static_assert(std::is_trivially_copyable_v<CharPtrPaddedFunctor>); + static_assert(std::is_trivially_copyable_v<PtrCharPaddedFunctor>); + static_assert( + absl::internal_any_invocable::IsStoredLocally<CharIntPaddedFunctor>()); + static_assert( + absl::internal_any_invocable::IsStoredLocally<CharPtrPaddedFunctor>()); + static_assert( + absl::internal_any_invocable::IsStoredLocally<PtrCharPaddedFunctor>()); + + const int target = 100; + { + CharIntPaddedFunctor f = MakeCharIntPadded(3, 40); + absl::AnyInvocable<int(int) const> a(f); + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(5), 48); + } + { + CharPtrPaddedFunctor f = MakeCharPtrPadded(4, &target); + absl::AnyInvocable<int(int) const> a(f); + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(5), 109); + } + { + PtrCharPaddedFunctor f = MakePtrCharPadded(&target, 6); + absl::AnyInvocable<int(int) const> a(f); + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(5), 111); + } +} + +struct MoveOnlySingleWordFunctor { + int32_t value; + + explicit MoveOnlySingleWordFunctor(int32_t v) noexcept : value(v) {} + MoveOnlySingleWordFunctor(const MoveOnlySingleWordFunctor&) = delete; + MoveOnlySingleWordFunctor& operator=(const MoveOnlySingleWordFunctor&) = + delete; + MoveOnlySingleWordFunctor(MoveOnlySingleWordFunctor&&) noexcept = default; + MoveOnlySingleWordFunctor& operator=(MoveOnlySingleWordFunctor&&) noexcept = + default; + + int operator()(int x) const noexcept { return value + x; } +}; + +struct MoveOnlyMultiWordFunctor { + const int* ptr; + int32_t value; + + MoveOnlyMultiWordFunctor(const int* p, int32_t v) noexcept + : ptr(p), value(v) {} + MoveOnlyMultiWordFunctor(const MoveOnlyMultiWordFunctor&) = delete; + MoveOnlyMultiWordFunctor& operator=(const MoveOnlyMultiWordFunctor&) = delete; + MoveOnlyMultiWordFunctor(MoveOnlyMultiWordFunctor&&) noexcept = default; + MoveOnlyMultiWordFunctor& operator=(MoveOnlyMultiWordFunctor&&) noexcept = + default; + + int operator()(int x) const noexcept { return *ptr + value + x; } +}; + +TEST(AnyInvTestTrivialStorage, MoveOnlyTriviallyCopyable) { + static_assert(std::is_trivially_copyable_v<MoveOnlySingleWordFunctor>); + static_assert(!std::is_copy_constructible_v<MoveOnlySingleWordFunctor>); + static_assert(absl::internal_any_invocable::IsStoredLocally< + MoveOnlySingleWordFunctor>()); + + static_assert(std::is_trivially_copyable_v<MoveOnlyMultiWordFunctor>); + static_assert(!std::is_copy_constructible_v<MoveOnlyMultiWordFunctor>); + static_assert(absl::internal_any_invocable::IsStoredLocally< + MoveOnlyMultiWordFunctor>()); + + { + absl::AnyInvocable<int(int) const> a(MoveOnlySingleWordFunctor(11)); + EXPECT_EQ(a(4), 15); + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(6), 17); + absl::AnyInvocable<int(int) const> c( + std::in_place_type<MoveOnlySingleWordFunctor>, 21); + b.swap(c); + EXPECT_EQ(b(1), 22); + EXPECT_EQ(c(1), 12); + } + + const int base = 100; + { + absl::AnyInvocable<int(int) const> a(MoveOnlyMultiWordFunctor(&base, 11)); + EXPECT_EQ(a(4), 115); + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(6), 117); + absl::AnyInvocable<int(int) const> c( + std::in_place_type<MoveOnlyMultiWordFunctor>, &base, 21); + b.swap(c); + EXPECT_EQ(b(1), 122); + EXPECT_EQ(c(1), 112); + } +} + +TEST(AnyInvTestTrivialStorage, EmptyCallable) { + struct EmptyFunctor { + int operator()(int x) const noexcept { return x + 7; } + }; + static_assert(std::is_empty_v<EmptyFunctor>); + static_assert(std::is_trivially_copyable_v<EmptyFunctor>); + static_assert(absl::internal_any_invocable::IsStoredLocally<EmptyFunctor>()); + + absl::AnyInvocable<int(int) const> a(EmptyFunctor{}); + EXPECT_EQ(a(10), 17); + absl::AnyInvocable<int(int) const> b(std::move(a)); + EXPECT_EQ(b(20), 27); + absl::AnyInvocable<int(int) const> c; + b.swap(c); + EXPECT_EQ(b, nullptr); + EXPECT_EQ(c(30), 37); +} + +TEST(AnyInvTestTrivialStorage, MutableSubWordAndMultiWordState) { + // 1-byte mutable capture. + { + std::uint8_t count = 1; + absl::AnyInvocable<int()> a = [count]() mutable { + count = static_cast<std::uint8_t>(count + 2); + return static_cast<int>(count); + }; + EXPECT_EQ(a(), 3); + absl::AnyInvocable<int()> b = std::move(a); + EXPECT_EQ(b(), 5); + absl::AnyInvocable<int()> c = [x = std::uint8_t{10}]() mutable { + x = static_cast<std::uint8_t>(x + 1); + return static_cast<int>(x); + }; + b.swap(c); + EXPECT_EQ(b(), 11); + EXPECT_EQ(c(), 7); + } + + // 4-byte mutable capture. + { + int32_t count = 10; + absl::AnyInvocable<int()> a = [count]() mutable { + count += 3; + return count; + }; + EXPECT_EQ(a(), 13); + absl::AnyInvocable<int()> b = std::move(a); + EXPECT_EQ(b(), 16); + } + + // Multi-word mutable capture (pointer + sub-word int). + { + int scale = 2; + int32_t count = 1; + absl::AnyInvocable<int()> a = [&scale, count]() mutable { + count += 1; + return count * scale; + }; + EXPECT_EQ(a(), 4); + absl::AnyInvocable<int()> b = std::move(a); + EXPECT_EQ(b(), 6); + } +} + +} // namespace INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestBasic, TestParameterListRemoteNonMovable);
diff --git a/absl/functional/internal/any_invocable.h b/absl/functional/internal/any_invocable.h index 1ff8bb6..6c16a88 100644 --- a/absl/functional/internal/any_invocable.h +++ b/absl/functional/internal/any_invocable.h
@@ -204,33 +204,80 @@ using InvokerType = ReturnType( TypeErasedState*, ForwardedParameterType<P>...) noexcept(SigIsNoexcept); -// The manager that is used when AnyInvocable is empty -inline void EmptyManager(FunctionToCall /*operation*/, - TypeErasedState* /*from*/, - TypeErasedState* /*to*/) noexcept {} +// The manager that is used when AnyInvocable is empty or holds an empty +// trivially copyable target object in local storage. +inline void EmptyManager(FunctionToCall operation, TypeErasedState* /*from*/, + TypeErasedState* const to) noexcept { + switch (operation) { + case FunctionToCall::relocate_from_to: + case FunctionToCall::relocate_from_to_and_query_rust: + // Begin the lifetime of `to->storage` so relocating an empty trivially + // copyable target object implicitly creates it in `to->storage` per P0593 + // ([intro.object]). Compiles to a no-op. + ::new (static_cast<void*>(&to->storage)) unsigned char[kStorageSize]; + return; + case FunctionToCall::dispose: + return; + } +} -// The manager that is used when a target function is in local storage and is -// a trivially copyable type. -inline void LocalManagerTrivial(FunctionToCall /*operation*/, +// The manager that is used when a non-empty trivially copyable target function +// fits in a single pointer-sized word of local storage. +inline void LocalManagerTrivialSingleWord(FunctionToCall operation, + TypeErasedState* const from, + TypeErasedState* const to) noexcept { + switch (operation) { + case FunctionToCall::relocate_from_to: + case FunctionToCall::relocate_from_to_and_query_rust: { + // Note: Correctness here relies on http://wg21.link/p0593 + // ([intro.object], [basic.types.general]). Placement-new of + // `unsigned char[kStorageSize]` activates `to->storage` and implicitly + // creates the target object within it; `memmove` then copies its single + // pointer-sized word without touching the unused second word. + unsigned char* dst = + ::new (static_cast<void*>(&to->storage)) unsigned char[kStorageSize]; + std::memmove(dst, from->storage, sizeof(void*)); + return; + } + case FunctionToCall::dispose: + return; + } +} + +// The manager that is used when a target function is in local storage, is a +// trivially copyable type, and spans more than one pointer-sized word. +inline void LocalManagerTrivial(FunctionToCall operation, TypeErasedState* const from, TypeErasedState* const to) noexcept { - // This single statement without branching handles both possible operations. - // - // For FunctionToCall::dispose, "from" and "to" point to the same state, and - // so this assignment logically would do nothing. - // - // Note: Correctness here relies on http://wg21.link/p0593, which has only - // become standard in C++20, though implementations do not break it in - // practice for earlier versions of C++. - // - // The correct way to do this without that paper is to first placement-new a - // default-constructed T in "to->storage" prior to the memmove, but doing so - // requires a different function to be created for each T that is stored - // locally, which can cause unnecessary bloat and be less cache friendly. - *to = *from; - - // Note: Because the type is trivially copyable, the destructor does not need - // to be called ("trivially copyable" requires a trivial destructor). + switch (operation) { + case FunctionToCall::relocate_from_to: + case FunctionToCall::relocate_from_to_and_query_rust: { + // Note: Correctness here relies on http://wg21.link/p0593, which has only + // become standard in C++20, though implementations do not break it in + // practice for earlier versions of C++. + // + // The correct way to do this without that paper is to first placement-new + // a default-constructed T in "to->storage" prior to the memmove, but + // doing so requires a different function to be created for each T that is + // stored locally, which can cause unnecessary bloat and be less cache + // friendly. + unsigned char* dst = + ::new (static_cast<void*>(&to->storage)) unsigned char[kStorageSize]; + // Copy as two pointer-sized words rather than a single 2-word copy so + // compilers emit word-sized scalar loads/stores instead of a 2-word SIMD + // load/store (avoiding store-to-load forwarding stalls with scalar writes + // before relocation and scalar reads in `LocalInvoker`). + constexpr std::size_t kWordSize = sizeof(void*); + static_assert(kStorageSize == 2 * kWordSize); + std::memmove(dst, from->storage, kWordSize); + std::memmove(dst + kWordSize, from->storage + kWordSize, kWordSize); + return; + } + case FunctionToCall::dispose: + // Note: Because the type is trivially copyable, the destructor does not + // need to be called ("trivially copyable" requires a trivial destructor). + return; + } } // The manager that is used when a target function is in local storage and is @@ -285,7 +332,13 @@ case FunctionToCall::relocate_from_to: case FunctionToCall::relocate_from_to_and_query_rust: // NOTE: Requires that the left-hand operand is already empty. - to->remote = from->remote; + // Copy `target` and `size` field-by-field rather than via whole-struct + // assignment (`to->remote = from->remote`) so compilers emit scalar + // pointer-sized loads/stores instead of a 2-word SIMD load/store, + // avoiding store-to-load forwarding stalls with scalar accesses in + // `InitializeRemoteManager`, `RemoteInvoker`, and `dispose`. + to->remote.target = from->remote.target; + to->remote.size = from->remote.size; return; case FunctionToCall::dispose: #if defined(__cpp_sized_deallocation) @@ -510,15 +563,39 @@ void InitializeStorage(Args&&... args) { using RawT = RemoveCVRef<QualTRef>; if constexpr (IsStoredLocally<RawT>()) { - ::new (static_cast<void*>(&state_.storage)) - RawT(std::forward<Args>(args)...); - invoker_ = LocalInvoker<SigIsNoexcept, ReturnType, QualTRef, P...>; // We can simplify our manager if we know the type is trivially copyable. if constexpr (std::is_trivially_copyable_v<RawT>) { - manager_ = LocalManagerTrivial; + if constexpr (std::is_empty_v<RawT>) { + ::new (static_cast<void*>(&state_.storage)) + RawT(std::forward<Args>(args)...); + manager_ = EmptyManager; + } else { + // Stage non-empty trivially copyable callables through a zeroed + // word-multiple buffer into `state_.storage` so narrow sub-word + // stores do not stall word-sized loads in `LocalManagerTrivial*`. + constexpr std::size_t kWordSize = sizeof(void*); + constexpr std::size_t kBufSize = + sizeof(RawT) <= kWordSize ? kWordSize : kStorageSize; + alignas(void*) alignas(RawT) unsigned char buf[kBufSize] = {}; + ::new (static_cast<void*>(buf)) RawT(std::forward<Args>(args)...); + unsigned char* dst = ::new ( + static_cast<void*>(&state_.storage)) unsigned char[kStorageSize]; + std::memcpy(dst, buf, kWordSize); + if constexpr (sizeof(RawT) <= kWordSize) { + manager_ = LocalManagerTrivialSingleWord; + } else { + std::memcpy(dst + kWordSize, buf + kWordSize, kWordSize); + manager_ = LocalManagerTrivial; + } + } } else { + ::new (static_cast<void*>(&state_.storage)) + RawT(std::forward<Args>(args)...); manager_ = LocalManagerNontrivial<RawT>; } + // This is set after everything else in case an exception is thrown in an + // earlier step of the initialization. + invoker_ = LocalInvoker<SigIsNoexcept, ReturnType, QualTRef, P...>; } else { InitializeRemoteManager<RawT>(std::forward<Args>(args)...); // This is set after everything else in case an exception is thrown in an