Copy AnyInvocable trivial storage in pointer-sized words and skip copies on dispose to avoid store-forwarding stalls

TypeErasedState spans two pointer-sized words (16 bytes on 64-bit platforms, aligned to 16 bytes on 64-bit SysV/Darwin), so `*to = *from` in LocalManagerTrivial() and `to->remote = from->remote` in RemoteManagerTrivial() compile to a 2-word SIMD load/store (such as `vmovaps (%rsi), %xmm0` / `vmovaps %xmm0, (%rdx)` on x86-64 and `ldr q0, [x1]` / `str q0, [x2]` on AArch64). When an absl::AnyInvocable is constructed from a function pointer, single-pointer lambda, sub-word scalar capture, or two-word remote state and then moved or destroyed (LocalManagerTrivial() previously self-copied on FunctionToCall::dispose as well), the 2-word SIMD load spans narrower scalar stores—or a 2-word SIMD store precedes scalar reads in LocalInvoker/RemoteInvoker—causing store-to-load forwarding stalls.

Avoid 2-word SIMD copies and unnecessary teardown copies using standard C++ without changing TypeErasedState's triviality or implicit-lifetime properties:
- Return early on FunctionToCall::dispose in LocalManagerTrivial() and LocalManagerTrivialSingleWord() (matching RemoteManagerTrivial()) to eliminate teardown loads/stores.
- Route empty trivially copyable local callables to EmptyManager() (starting `to->storage` lifetime per P0593 as a no-op), single-word (`<= sizeof(void*)`) callables to LocalManagerTrivialSingleWord() (single pointer-sized std::memmove), and multi-word callables to LocalManagerTrivial() (two sequential pointer-sized std::memmove calls).
- Copy `remote.target` and `remote.size` field-by-field in RemoteManagerTrivial() instead of via whole-struct assignment.
- Stage non-empty trivially copyable local callables through a zeroed word-multiple `unsigned char` buffer into `state_.storage` via pointer-sized std::memcpy calls in InitializeStorage() so narrow sub-word stores do not stall subsequent word-sized loads.
- Guard TypeErasedState triviality with static_assert and add unit tests covering empty, sub-word, partial/full multi-word, padded, mutable, move-only, and remote trivial callables.

PiperOrigin-RevId: 1019520730
diff --git a/absl/functional/any_invocable_test.h b/absl/functional/any_invocable_test.h
index 6d09c12..5c0a36c 100644
--- a/absl/functional/any_invocable_test.h
+++ b/absl/functional/any_invocable_test.h
@@ -44,6 +44,12 @@
 static_assert(absl::internal_any_invocable::kStorageSize >= sizeof(void*),
               "These tests assume that the small object storage is at least "
               "the size of a pointer.");
+static_assert(
+    std::is_trivially_copyable_v<absl::internal_any_invocable::TypeErasedState>,
+    "TypeErasedState must remain trivially copyable.");
+static_assert(std::is_trivially_default_constructible_v<
+                  absl::internal_any_invocable::TypeErasedState>,
+              "TypeErasedState must remain trivially default-constructible.");
 
 namespace absl_any_invocable_test {
 
diff --git a/absl/functional/any_invocable_test_inst4.cc b/absl/functional/any_invocable_test_inst4.cc
index ac21ca9..f5fbcf6 100644
--- a/absl/functional/any_invocable_test_inst4.cc
+++ b/absl/functional/any_invocable_test_inst4.cc
@@ -18,10 +18,303 @@
 
 // SKIP_ABSL_INLINE_NAMESPACE_CHECK
 
+#include <cstddef>
+#include <cstdint>
+#include <type_traits>
+#include <utility>
+
 #include "gtest/gtest.h"
+#include "absl/base/attributes.h"
+#include "absl/functional/any_invocable.h"
 #include "absl/functional/any_invocable_test.h"
 
 namespace absl_any_invocable_test {
+namespace {
+
+template <size_t N>
+struct ByteArrayFunctor {
+  unsigned char bytes[N];
+
+  int operator()(int seed) const noexcept {
+    int sum = seed;
+    for (size_t i = 0; i < N; ++i) {
+      sum += static_cast<int>(bytes[i]) * static_cast<int>(i + 1);
+    }
+    return sum;
+  }
+};
+
+template <size_t N>
+void VerifyByteArrayFunctor() {
+  static_assert(std::is_trivially_copyable_v<ByteArrayFunctor<N>>);
+  if constexpr (N <= absl::internal_any_invocable::kStorageSize) {
+    static_assert(
+        absl::internal_any_invocable::IsStoredLocally<ByteArrayFunctor<N>>());
+  }
+
+  ByteArrayFunctor<N> f1{};
+  ByteArrayFunctor<N> f2{};
+  for (size_t i = 0; i < N; ++i) {
+    f1.bytes[i] = static_cast<unsigned char>(i + 3);
+    f2.bytes[i] = static_cast<unsigned char>(i + 17);
+  }
+  const int expected1 = f1(5);
+  const int expected2 = f2(5);
+
+  absl::AnyInvocable<int(int) const> a(f1);
+  EXPECT_EQ(a(5), expected1);
+
+  absl::AnyInvocable<int(int) const> b(std::move(a));
+  EXPECT_EQ(b(5), expected1);
+
+  absl::AnyInvocable<int(int) const> c(std::in_place_type<ByteArrayFunctor<N>>,
+                                       f2);
+  EXPECT_EQ(c(5), expected2);
+
+  b.swap(c);
+  EXPECT_EQ(b(5), expected2);
+  EXPECT_EQ(c(5), expected1);
+
+  b = std::move(c);
+  EXPECT_EQ(b(5), expected1);
+  b = nullptr;
+  EXPECT_EQ(b, nullptr);
+}
+
+TEST(AnyInvTestTrivialStorage, SubWordAndMultiWordSizes) {
+  VerifyByteArrayFunctor<1>();
+  VerifyByteArrayFunctor<2>();
+  VerifyByteArrayFunctor<3>();
+  VerifyByteArrayFunctor<4>();
+  VerifyByteArrayFunctor<5>();
+  VerifyByteArrayFunctor<6>();
+  VerifyByteArrayFunctor<7>();
+  VerifyByteArrayFunctor<8>();
+  VerifyByteArrayFunctor<9>();
+  VerifyByteArrayFunctor<12>();
+  VerifyByteArrayFunctor<15>();
+  VerifyByteArrayFunctor<16>();
+  // Also exercise remote trivially destructible storage
+  // (`RemoteManagerTrivial`) with default alignment (alignof == 1 <=
+  // ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT).
+  VerifyByteArrayFunctor<24>();
+}
+
+struct CharIntPaddedFunctor {
+  char c;
+  int x;
+
+  int operator()(int delta) const noexcept {
+    return static_cast<int>(c) + x + delta;
+  }
+};
+
+struct CharPtrPaddedFunctor {
+  char c;
+  const int* p;
+
+  int operator()(int delta) const noexcept {
+    return static_cast<int>(c) + *p + delta;
+  }
+};
+
+struct PtrCharPaddedFunctor {
+  const int* p;
+  char c;
+
+  int operator()(int delta) const noexcept {
+    return static_cast<int>(c) + *p + delta;
+  }
+};
+
+// Construct padded structs field-by-field in noinline helpers so padding bytes
+// remain uninitialized under sanitizers such as MSan.
+ABSL_ATTRIBUTE_NOINLINE CharIntPaddedFunctor MakeCharIntPadded(char c, int x) {
+  CharIntPaddedFunctor f;
+  f.c = c;
+  f.x = x;
+  return f;
+}
+
+ABSL_ATTRIBUTE_NOINLINE CharPtrPaddedFunctor MakeCharPtrPadded(char c,
+                                                               const int* p) {
+  CharPtrPaddedFunctor f;
+  f.c = c;
+  f.p = p;
+  return f;
+}
+
+ABSL_ATTRIBUTE_NOINLINE PtrCharPaddedFunctor MakePtrCharPadded(const int* p,
+                                                               char c) {
+  PtrCharPaddedFunctor f;
+  f.p = p;
+  f.c = c;
+  return f;
+}
+
+TEST(AnyInvTestTrivialStorage, InternalAndTailPadding) {
+  static_assert(std::is_trivially_copyable_v<CharIntPaddedFunctor>);
+  static_assert(std::is_trivially_copyable_v<CharPtrPaddedFunctor>);
+  static_assert(std::is_trivially_copyable_v<PtrCharPaddedFunctor>);
+  static_assert(
+      absl::internal_any_invocable::IsStoredLocally<CharIntPaddedFunctor>());
+  static_assert(
+      absl::internal_any_invocable::IsStoredLocally<CharPtrPaddedFunctor>());
+  static_assert(
+      absl::internal_any_invocable::IsStoredLocally<PtrCharPaddedFunctor>());
+
+  const int target = 100;
+  {
+    CharIntPaddedFunctor f = MakeCharIntPadded(3, 40);
+    absl::AnyInvocable<int(int) const> a(f);
+    absl::AnyInvocable<int(int) const> b(std::move(a));
+    EXPECT_EQ(b(5), 48);
+  }
+  {
+    CharPtrPaddedFunctor f = MakeCharPtrPadded(4, &target);
+    absl::AnyInvocable<int(int) const> a(f);
+    absl::AnyInvocable<int(int) const> b(std::move(a));
+    EXPECT_EQ(b(5), 109);
+  }
+  {
+    PtrCharPaddedFunctor f = MakePtrCharPadded(&target, 6);
+    absl::AnyInvocable<int(int) const> a(f);
+    absl::AnyInvocable<int(int) const> b(std::move(a));
+    EXPECT_EQ(b(5), 111);
+  }
+}
+
+struct MoveOnlySingleWordFunctor {
+  int32_t value;
+
+  explicit MoveOnlySingleWordFunctor(int32_t v) noexcept : value(v) {}
+  MoveOnlySingleWordFunctor(const MoveOnlySingleWordFunctor&) = delete;
+  MoveOnlySingleWordFunctor& operator=(const MoveOnlySingleWordFunctor&) =
+      delete;
+  MoveOnlySingleWordFunctor(MoveOnlySingleWordFunctor&&) noexcept = default;
+  MoveOnlySingleWordFunctor& operator=(MoveOnlySingleWordFunctor&&) noexcept =
+      default;
+
+  int operator()(int x) const noexcept { return value + x; }
+};
+
+struct MoveOnlyMultiWordFunctor {
+  const int* ptr;
+  int32_t value;
+
+  MoveOnlyMultiWordFunctor(const int* p, int32_t v) noexcept
+      : ptr(p), value(v) {}
+  MoveOnlyMultiWordFunctor(const MoveOnlyMultiWordFunctor&) = delete;
+  MoveOnlyMultiWordFunctor& operator=(const MoveOnlyMultiWordFunctor&) = delete;
+  MoveOnlyMultiWordFunctor(MoveOnlyMultiWordFunctor&&) noexcept = default;
+  MoveOnlyMultiWordFunctor& operator=(MoveOnlyMultiWordFunctor&&) noexcept =
+      default;
+
+  int operator()(int x) const noexcept { return *ptr + value + x; }
+};
+
+TEST(AnyInvTestTrivialStorage, MoveOnlyTriviallyCopyable) {
+  static_assert(std::is_trivially_copyable_v<MoveOnlySingleWordFunctor>);
+  static_assert(!std::is_copy_constructible_v<MoveOnlySingleWordFunctor>);
+  static_assert(absl::internal_any_invocable::IsStoredLocally<
+                MoveOnlySingleWordFunctor>());
+
+  static_assert(std::is_trivially_copyable_v<MoveOnlyMultiWordFunctor>);
+  static_assert(!std::is_copy_constructible_v<MoveOnlyMultiWordFunctor>);
+  static_assert(absl::internal_any_invocable::IsStoredLocally<
+                MoveOnlyMultiWordFunctor>());
+
+  {
+    absl::AnyInvocable<int(int) const> a(MoveOnlySingleWordFunctor(11));
+    EXPECT_EQ(a(4), 15);
+    absl::AnyInvocable<int(int) const> b(std::move(a));
+    EXPECT_EQ(b(6), 17);
+    absl::AnyInvocable<int(int) const> c(
+        std::in_place_type<MoveOnlySingleWordFunctor>, 21);
+    b.swap(c);
+    EXPECT_EQ(b(1), 22);
+    EXPECT_EQ(c(1), 12);
+  }
+
+  const int base = 100;
+  {
+    absl::AnyInvocable<int(int) const> a(MoveOnlyMultiWordFunctor(&base, 11));
+    EXPECT_EQ(a(4), 115);
+    absl::AnyInvocable<int(int) const> b(std::move(a));
+    EXPECT_EQ(b(6), 117);
+    absl::AnyInvocable<int(int) const> c(
+        std::in_place_type<MoveOnlyMultiWordFunctor>, &base, 21);
+    b.swap(c);
+    EXPECT_EQ(b(1), 122);
+    EXPECT_EQ(c(1), 112);
+  }
+}
+
+TEST(AnyInvTestTrivialStorage, EmptyCallable) {
+  struct EmptyFunctor {
+    int operator()(int x) const noexcept { return x + 7; }
+  };
+  static_assert(std::is_empty_v<EmptyFunctor>);
+  static_assert(std::is_trivially_copyable_v<EmptyFunctor>);
+  static_assert(absl::internal_any_invocable::IsStoredLocally<EmptyFunctor>());
+
+  absl::AnyInvocable<int(int) const> a(EmptyFunctor{});
+  EXPECT_EQ(a(10), 17);
+  absl::AnyInvocable<int(int) const> b(std::move(a));
+  EXPECT_EQ(b(20), 27);
+  absl::AnyInvocable<int(int) const> c;
+  b.swap(c);
+  EXPECT_EQ(b, nullptr);
+  EXPECT_EQ(c(30), 37);
+}
+
+TEST(AnyInvTestTrivialStorage, MutableSubWordAndMultiWordState) {
+  // 1-byte mutable capture.
+  {
+    std::uint8_t count = 1;
+    absl::AnyInvocable<int()> a = [count]() mutable {
+      count = static_cast<std::uint8_t>(count + 2);
+      return static_cast<int>(count);
+    };
+    EXPECT_EQ(a(), 3);
+    absl::AnyInvocable<int()> b = std::move(a);
+    EXPECT_EQ(b(), 5);
+    absl::AnyInvocable<int()> c = [x = std::uint8_t{10}]() mutable {
+      x = static_cast<std::uint8_t>(x + 1);
+      return static_cast<int>(x);
+    };
+    b.swap(c);
+    EXPECT_EQ(b(), 11);
+    EXPECT_EQ(c(), 7);
+  }
+
+  // 4-byte mutable capture.
+  {
+    int32_t count = 10;
+    absl::AnyInvocable<int()> a = [count]() mutable {
+      count += 3;
+      return count;
+    };
+    EXPECT_EQ(a(), 13);
+    absl::AnyInvocable<int()> b = std::move(a);
+    EXPECT_EQ(b(), 16);
+  }
+
+  // Multi-word mutable capture (pointer + sub-word int).
+  {
+    int scale = 2;
+    int32_t count = 1;
+    absl::AnyInvocable<int()> a = [&scale, count]() mutable {
+      count += 1;
+      return count * scale;
+    };
+    EXPECT_EQ(a(), 4);
+    absl::AnyInvocable<int()> b = std::move(a);
+    EXPECT_EQ(b(), 6);
+  }
+}
+
+}  // namespace
 
 INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestBasic,
                                TestParameterListRemoteNonMovable);
diff --git a/absl/functional/internal/any_invocable.h b/absl/functional/internal/any_invocable.h
index 1ff8bb6..6c16a88 100644
--- a/absl/functional/internal/any_invocable.h
+++ b/absl/functional/internal/any_invocable.h
@@ -204,33 +204,80 @@
 using InvokerType = ReturnType(
     TypeErasedState*, ForwardedParameterType<P>...) noexcept(SigIsNoexcept);
 
-// The manager that is used when AnyInvocable is empty
-inline void EmptyManager(FunctionToCall /*operation*/,
-                         TypeErasedState* /*from*/,
-                         TypeErasedState* /*to*/) noexcept {}
+// The manager that is used when AnyInvocable is empty or holds an empty
+// trivially copyable target object in local storage.
+inline void EmptyManager(FunctionToCall operation, TypeErasedState* /*from*/,
+                         TypeErasedState* const to) noexcept {
+  switch (operation) {
+    case FunctionToCall::relocate_from_to:
+    case FunctionToCall::relocate_from_to_and_query_rust:
+      // Begin the lifetime of `to->storage` so relocating an empty trivially
+      // copyable target object implicitly creates it in `to->storage` per P0593
+      // ([intro.object]). Compiles to a no-op.
+      ::new (static_cast<void*>(&to->storage)) unsigned char[kStorageSize];
+      return;
+    case FunctionToCall::dispose:
+      return;
+  }
+}
 
-// The manager that is used when a target function is in local storage and is
-// a trivially copyable type.
-inline void LocalManagerTrivial(FunctionToCall /*operation*/,
+// The manager that is used when a non-empty trivially copyable target function
+// fits in a single pointer-sized word of local storage.
+inline void LocalManagerTrivialSingleWord(FunctionToCall operation,
+                                          TypeErasedState* const from,
+                                          TypeErasedState* const to) noexcept {
+  switch (operation) {
+    case FunctionToCall::relocate_from_to:
+    case FunctionToCall::relocate_from_to_and_query_rust: {
+      // Note: Correctness here relies on http://wg21.link/p0593
+      // ([intro.object], [basic.types.general]). Placement-new of
+      // `unsigned char[kStorageSize]` activates `to->storage` and implicitly
+      // creates the target object within it; `memmove` then copies its single
+      // pointer-sized word without touching the unused second word.
+      unsigned char* dst =
+          ::new (static_cast<void*>(&to->storage)) unsigned char[kStorageSize];
+      std::memmove(dst, from->storage, sizeof(void*));
+      return;
+    }
+    case FunctionToCall::dispose:
+      return;
+  }
+}
+
+// The manager that is used when a target function is in local storage, is a
+// trivially copyable type, and spans more than one pointer-sized word.
+inline void LocalManagerTrivial(FunctionToCall operation,
                                 TypeErasedState* const from,
                                 TypeErasedState* const to) noexcept {
-  // This single statement without branching handles both possible operations.
-  //
-  // For FunctionToCall::dispose, "from" and "to" point to the same state, and
-  // so this assignment logically would do nothing.
-  //
-  // Note: Correctness here relies on http://wg21.link/p0593, which has only
-  // become standard in C++20, though implementations do not break it in
-  // practice for earlier versions of C++.
-  //
-  // The correct way to do this without that paper is to first placement-new a
-  // default-constructed T in "to->storage" prior to the memmove, but doing so
-  // requires a different function to be created for each T that is stored
-  // locally, which can cause unnecessary bloat and be less cache friendly.
-  *to = *from;
-
-  // Note: Because the type is trivially copyable, the destructor does not need
-  // to be called ("trivially copyable" requires a trivial destructor).
+  switch (operation) {
+    case FunctionToCall::relocate_from_to:
+    case FunctionToCall::relocate_from_to_and_query_rust: {
+      // Note: Correctness here relies on http://wg21.link/p0593, which has only
+      // become standard in C++20, though implementations do not break it in
+      // practice for earlier versions of C++.
+      //
+      // The correct way to do this without that paper is to first placement-new
+      // a default-constructed T in "to->storage" prior to the memmove, but
+      // doing so requires a different function to be created for each T that is
+      // stored locally, which can cause unnecessary bloat and be less cache
+      // friendly.
+      unsigned char* dst =
+          ::new (static_cast<void*>(&to->storage)) unsigned char[kStorageSize];
+      // Copy as two pointer-sized words rather than a single 2-word copy so
+      // compilers emit word-sized scalar loads/stores instead of a 2-word SIMD
+      // load/store (avoiding store-to-load forwarding stalls with scalar writes
+      // before relocation and scalar reads in `LocalInvoker`).
+      constexpr std::size_t kWordSize = sizeof(void*);
+      static_assert(kStorageSize == 2 * kWordSize);
+      std::memmove(dst, from->storage, kWordSize);
+      std::memmove(dst + kWordSize, from->storage + kWordSize, kWordSize);
+      return;
+    }
+    case FunctionToCall::dispose:
+      // Note: Because the type is trivially copyable, the destructor does not
+      // need to be called ("trivially copyable" requires a trivial destructor).
+      return;
+  }
 }
 
 // The manager that is used when a target function is in local storage and is
@@ -285,7 +332,13 @@
     case FunctionToCall::relocate_from_to:
     case FunctionToCall::relocate_from_to_and_query_rust:
       // NOTE: Requires that the left-hand operand is already empty.
-      to->remote = from->remote;
+      // Copy `target` and `size` field-by-field rather than via whole-struct
+      // assignment (`to->remote = from->remote`) so compilers emit scalar
+      // pointer-sized loads/stores instead of a 2-word SIMD load/store,
+      // avoiding store-to-load forwarding stalls with scalar accesses in
+      // `InitializeRemoteManager`, `RemoteInvoker`, and `dispose`.
+      to->remote.target = from->remote.target;
+      to->remote.size = from->remote.size;
       return;
     case FunctionToCall::dispose:
 #if defined(__cpp_sized_deallocation)
@@ -510,15 +563,39 @@
   void InitializeStorage(Args&&... args) {
     using RawT = RemoveCVRef<QualTRef>;
     if constexpr (IsStoredLocally<RawT>()) {
-      ::new (static_cast<void*>(&state_.storage))
-          RawT(std::forward<Args>(args)...);
-      invoker_ = LocalInvoker<SigIsNoexcept, ReturnType, QualTRef, P...>;
       // We can simplify our manager if we know the type is trivially copyable.
       if constexpr (std::is_trivially_copyable_v<RawT>) {
-        manager_ = LocalManagerTrivial;
+        if constexpr (std::is_empty_v<RawT>) {
+          ::new (static_cast<void*>(&state_.storage))
+              RawT(std::forward<Args>(args)...);
+          manager_ = EmptyManager;
+        } else {
+          // Stage non-empty trivially copyable callables through a zeroed
+          // word-multiple buffer into `state_.storage` so narrow sub-word
+          // stores do not stall word-sized loads in `LocalManagerTrivial*`.
+          constexpr std::size_t kWordSize = sizeof(void*);
+          constexpr std::size_t kBufSize =
+              sizeof(RawT) <= kWordSize ? kWordSize : kStorageSize;
+          alignas(void*) alignas(RawT) unsigned char buf[kBufSize] = {};
+          ::new (static_cast<void*>(buf)) RawT(std::forward<Args>(args)...);
+          unsigned char* dst = ::new (
+              static_cast<void*>(&state_.storage)) unsigned char[kStorageSize];
+          std::memcpy(dst, buf, kWordSize);
+          if constexpr (sizeof(RawT) <= kWordSize) {
+            manager_ = LocalManagerTrivialSingleWord;
+          } else {
+            std::memcpy(dst + kWordSize, buf + kWordSize, kWordSize);
+            manager_ = LocalManagerTrivial;
+          }
+        }
       } else {
+        ::new (static_cast<void*>(&state_.storage))
+            RawT(std::forward<Args>(args)...);
         manager_ = LocalManagerNontrivial<RawT>;
       }
+      // This is set after everything else in case an exception is thrown in an
+      // earlier step of the initialization.
+      invoker_ = LocalInvoker<SigIsNoexcept, ReturnType, QualTRef, P...>;
     } else {
       InitializeRemoteManager<RawT>(std::forward<Args>(args)...);
       // This is set after everything else in case an exception is thrown in an