Rollback of 22f1a4db4fd5b85f1899ff201e2aa938e5363a05: Skip indirect manager calls on empty and trivial-local `absl::AnyInvocable` move and destroy. PiperOrigin-RevId: 1022717868
diff --git a/absl/functional/any_invocable_test.h b/absl/functional/any_invocable_test.h index 2c16c60..6d09c12 100644 --- a/absl/functional/any_invocable_test.h +++ b/absl/functional/any_invocable_test.h
@@ -374,24 +374,12 @@ return static_cast<VoidThisParamType>(fun); } - using CompatibleQualifiers = - std::conditional_t<std::is_rvalue_reference_v<Qual>, const _&&, const _&>; - using CompatibleAnyInvocableFunType = - GiveQualifiersToFun<CompatibleQualifiers, UnqualifiedFunType>; + std::conditional_t<std::is_rvalue_reference_v<Qual>, + GiveQualifiersToFun<const _&&, UnqualifiedFunType>, + GiveQualifiersToFun<const _&, UnqualifiedFunType>>; using CompatibleAnyInvType = AnyInvocable<CompatibleAnyInvocableFunType>; - using CompatibleAddType = - add<kMovability, kDestructibility, CompatibleQualifiers, - kCallExceptionSpec, kSize, kAlignment>; - - using CompatibleNoexceptFunType = - GiveQualifiersToFun<CompatibleQualifiers, - Int(Int, const int&, int) noexcept>; - using CompatibleNoexceptAnyInvType = AnyInvocable<CompatibleNoexceptFunType>; - using CompatibleNoexceptAddType = - add<kMovability, kDestructibility, CompatibleQualifiers, NothrowCall::yes, - kSize, kAlignment>; using IncompatibleInvocable = std::conditional_t<std::is_rvalue_reference_v<Qual>, @@ -533,31 +521,16 @@ TYPED_TEST_P(AnyInvTestBasic, ConstructionCompatibleAnyInvocableNonempty) { using AnyInvType = typename TypeParam::AnyInvType; using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; - using CompatibleAddType = typename TypeParam::CompatibleAddType; - { - CompatibleAnyInvType other = &add_function; - AnyInvType fun = std::move(other); + CompatibleAnyInvType other = &add_function; + AnyInvType fun = std::move(other); - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_EQ(other, nullptr); // NOLINT - EXPECT_EQ(nullptr, other); // NOLINT + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_EQ(other, nullptr); // NOLINT + EXPECT_EQ(nullptr, other); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } - - { - CompatibleAnyInvType other(std::in_place_type<CompatibleAddType>, 5); - AnyInvType fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_EQ(other, nullptr); // NOLINT - EXPECT_EQ(nullptr, other); // NOLINT - - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); } TYPED_TEST_P(AnyInvTestBasic, ConversionToBool) { @@ -908,29 +881,15 @@ AssignCompatibleAnyInvocableEmptyLhsNonemptyRhs) { using AnyInvType = typename TypeParam::AnyInvType; using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; - using CompatibleAddType = typename TypeParam::CompatibleAddType; - { - CompatibleAnyInvType other = &add_function; - AnyInvType fun; - fun = std::move(other); + CompatibleAnyInvType other = &add_function; + AnyInvType fun; + fun = std::move(other); - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } - - { - CompatibleAnyInvType other(std::in_place_type<CompatibleAddType>, 5); - AnyInvType fun; - fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); } TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullptrNonemptyLhs) { @@ -1024,31 +983,16 @@ TYPED_TEST_P(AnyInvTestCombinatoric, AssignCompatibleAnyInvocableNonemptyLhsNonemptyRhs) { using AnyInvType = typename TypeParam::AnyInvType; - using AddType = typename TypeParam::AddType; using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; - using CompatibleAddType = typename TypeParam::CompatibleAddType; - { - CompatibleAnyInvType other = &add_function; - AnyInvType fun = &mult_function; - fun = std::move(other); + CompatibleAnyInvType other = &add_function; + AnyInvType fun = &mult_function; + fun = std::move(other); - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } - - { - CompatibleAnyInvType other(std::in_place_type<CompatibleAddType>, 5); - AnyInvType fun(std::in_place_type<AddType>, 20); - fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); } TYPED_TEST_P(AnyInvTestCombinatoric, SwapEmptyLhsEmptyRhs) { @@ -1259,10 +1203,6 @@ TYPED_TEST_P(AnyInvTestNoexceptFalse, ConversionConstructionConstraints) { using AnyInvType = typename TypeParam::AnyInvType; - using CompatibleNoexceptAnyInvType = - typename TypeParam::CompatibleNoexceptAnyInvType; - using CompatibleNoexceptAddType = - typename TypeParam::CompatibleNoexceptAddType; EXPECT_TRUE( (std::is_constructible_v< @@ -1270,34 +1210,10 @@ EXPECT_FALSE( (std::is_constructible_v<AnyInvType, typename TypeParam::IncompatibleInvocable>)); - - { - CompatibleNoexceptAnyInvType other = &add_function; - AnyInvType fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } - - { - CompatibleNoexceptAnyInvType other( - std::in_place_type<CompatibleNoexceptAddType>, 5); - AnyInvType fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } } TYPED_TEST_P(AnyInvTestNoexceptFalse, ConversionAssignConstraints) { using AnyInvType = typename TypeParam::AnyInvType; - using AddType = typename TypeParam::AddType; - using CompatibleNoexceptAnyInvType = - typename TypeParam::CompatibleNoexceptAnyInvType; - using CompatibleNoexceptAddType = - typename TypeParam::CompatibleNoexceptAddType; EXPECT_TRUE( (std::is_assignable_v< @@ -1305,27 +1221,6 @@ EXPECT_FALSE( (std::is_assignable_v<AnyInvType&, typename TypeParam::IncompatibleInvocable>)); - - { - CompatibleNoexceptAnyInvType other = &add_function; - AnyInvType fun = &mult_function; - fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } - - { - CompatibleNoexceptAnyInvType other( - std::in_place_type<CompatibleNoexceptAddType>, 5); - AnyInvType fun(std::in_place_type<AddType>, 20); - fun = std::move(other); - - EXPECT_FALSE(static_cast<bool>(other)); // NOLINT - EXPECT_TRUE(static_cast<bool>(fun)); - EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); - } } template <class T>
diff --git a/absl/functional/internal/any_invocable.h b/absl/functional/internal/any_invocable.h index 122babc..1ff8bb6 100644 --- a/absl/functional/internal/any_invocable.h +++ b/absl/functional/internal/any_invocable.h
@@ -38,12 +38,9 @@ // 3) A function pointer to an "invoker" function operation that invokes the // // target object, directly returning the result. // // // -// When in the logically empty state, the manager function is shared with // -// locally stored trivially copyable targets (LocalManagerTrivial, which // -// copies storage on move and is a no-op on dispose) so ~CoreImpl(), Clear(), // -// and RelocateFrom() can skip indirect calls for both cases with a single // -// pointer comparison, and the invoker function is one that would be // -// undefined behavior to call. // +// When in the logically empty state, the manager function is an empty // +// function and the invoker function is one that would be undefined behavior // +// to call. // // // // An additional optimization is performed when converting from one // // AnyInvocable to another where only the noexcept specification and/or the // @@ -207,60 +204,33 @@ using InvokerType = ReturnType( TypeErasedState*, ForwardedParameterType<P>...) noexcept(SigIsNoexcept); -// A no-op manager retained for external `rs_std::DynCallable` callers; -// `CoreImpl` uses `LocalManagerTrivial` for both empty and trivial-local -// states. +// The manager that is used when AnyInvocable is empty inline void EmptyManager(FunctionToCall /*operation*/, TypeErasedState* /*from*/, TypeErasedState* /*to*/) noexcept {} -// The manager that is used when `AnyInvocable` is empty or when a target -// function is in local storage and is a trivially copyable type. Sharing one -// manager allows `~CoreImpl()`, `Clear()`, and `RelocateFrom()` to skip -// indirect calls for both states with a single pointer comparison. -inline void LocalManagerTrivial(FunctionToCall operation, +// The manager that is used when a target function is in local storage and is +// a trivially copyable type. +inline void LocalManagerTrivial(FunctionToCall /*operation*/, TypeErasedState* const from, TypeErasedState* const to) noexcept { - switch (operation) { - case FunctionToCall::relocate_from_to: - case FunctionToCall::relocate_from_to_and_query_rust: - // Note: Correctness for locally stored trivially copyable types relies on - // http://wg21.link/p0593, which has only become standard in C++20, though - // implementations do not break it in practice for earlier versions of - // C++. - // - // The correct way to do this without that paper is to first placement-new - // a default-constructed T in "to->storage" prior to the memmove, but - // doing so requires a different function to be created for each T that is - // stored locally, which can cause unnecessary bloat and be less cache - // friendly. - // - // When AnyInvocable is empty or holds a locally stored callable with - // `sizeof(RawT) < kStorageSize` or padding, bytes of `*from` are left - // unwritten. `*to = *from` copies the `TypeErasedState` union, which is - // well-defined because trivial union copy-assignment copies the object - // representation as `unsigned char` bytes ([class.copy.assign]) rather - // than memberwise copying `remote.target`; suppress GCC's - // `-Wuninitialized` diagnostics when `LocalManagerTrivial` is inlined - // into `RelocateFrom`. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpragmas" -#pragma GCC diagnostic ignored "-Wuninitialized" -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - *to = *from; -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - return; - case FunctionToCall::dispose: - // Because the type is trivially copyable (or AnyInvocable is empty), the - // destructor does not need to be called ("trivially copyable" requires a - // trivial destructor). - return; - } - ABSL_UNREACHABLE(); + // This single statement without branching handles both possible operations. + // + // For FunctionToCall::dispose, "from" and "to" point to the same state, and + // so this assignment logically would do nothing. + // + // Note: Correctness here relies on http://wg21.link/p0593, which has only + // become standard in C++20, though implementations do not break it in + // practice for earlier versions of C++. + // + // The correct way to do this without that paper is to first placement-new a + // default-constructed T in "to->storage" prior to the memmove, but doing so + // requires a different function to be created for each T that is stored + // locally, which can cause unnecessary bloat and be less cache friendly. + *to = *from; + + // Note: Because the type is trivially copyable, the destructor does not need + // to be called ("trivially copyable" requires a trivial destructor). } // The manager that is used when a target function is in local storage and is @@ -435,7 +405,7 @@ public: using result_type = ReturnType; - CoreImpl() noexcept : manager_(LocalManagerTrivial), invoker_(nullptr) {} + CoreImpl() noexcept : manager_(EmptyManager), invoker_(nullptr) {} // Note: QualDecayedTRef here includes the cv-ref qualifiers associated with // the invocation of the Invocable. The unqualified type is the target object @@ -462,19 +432,24 @@ #if !defined(__clang__) && defined(__GNUC__) #pragma GCC diagnostic pop #endif - manager_ = LocalManagerTrivial; + manager_ = EmptyManager; invoker_ = nullptr; } else { InitializeStorage<QualDecayedTRef>(std::forward<F>(f)); } } else if constexpr (IsCompatibleAnyInvocable<DecayedT>::value) { // In this case we can "steal the guts" of the other AnyInvocable. - RelocateFrom(std::forward<F>(f)); + f.manager_(FunctionToCall::relocate_from_to, &f.state_, &state_); + manager_ = f.manager_; + invoker_ = f.invoker_; + + f.manager_ = EmptyManager; + f.invoker_ = nullptr; } else if constexpr (IsAnyInvocable<DecayedT>::value) { if (f.HasValue()) { InitializeStorage<QualDecayedTRef>(std::forward<F>(f)); } else { - manager_ = LocalManagerTrivial; + manager_ = EmptyManager; invoker_ = nullptr; } } else { @@ -490,7 +465,13 @@ InitializeStorage<QualTRef>(std::forward<Args>(args)...); } - CoreImpl(CoreImpl&& other) noexcept { RelocateFrom(std::move(other)); } + CoreImpl(CoreImpl&& other) noexcept { + other.manager_(FunctionToCall::relocate_from_to, &other.state_, &state_); + manager_ = other.manager_; + invoker_ = other.invoker_; + other.manager_ = EmptyManager; + other.invoker_ = nullptr; + } CoreImpl& operator=(CoreImpl&& other) noexcept { // Put the left-hand operand in an empty state. @@ -503,47 +484,27 @@ Clear(); // Perform the actual move/destroy operation on the target function. - RelocateFrom(std::move(other)); + other.manager_(FunctionToCall::relocate_from_to, &other.state_, &state_); + manager_ = other.manager_; + invoker_ = other.invoker_; + other.manager_ = EmptyManager; + other.invoker_ = nullptr; return *this; } - ~CoreImpl() { - if (manager_ != LocalManagerTrivial) { - manager_(FunctionToCall::dispose, &state_, &state_); - } - } + ~CoreImpl() { manager_(FunctionToCall::dispose, &state_, &state_); } // Check whether or not the AnyInvocable is in the empty state. bool HasValue() const { return invoker_ != nullptr; } // Effects: Puts the object into its empty state. void Clear() { - if (manager_ != LocalManagerTrivial) { - manager_(FunctionToCall::dispose, &state_, &state_); - manager_ = LocalManagerTrivial; - } + manager_(FunctionToCall::dispose, &state_, &state_); + manager_ = EmptyManager; invoker_ = nullptr; } - // Moves the target from `other` into `*this` (which must not hold an active - // target), leaving `other` empty. - template <bool OtherIsNoexcept> - void RelocateFrom( - CoreImpl<OtherIsNoexcept, ReturnType, P...>&& other) noexcept { - if (other.manager_ == LocalManagerTrivial) { - LocalManagerTrivial(FunctionToCall::relocate_from_to, &other.state_, - &state_); - manager_ = LocalManagerTrivial; - } else { - other.manager_(FunctionToCall::relocate_from_to, &other.state_, &state_); - manager_ = other.manager_; - other.manager_ = LocalManagerTrivial; - } - invoker_ = other.invoker_; - other.invoker_ = nullptr; - } - // Use local (inline) storage for applicable target object types. template <class QualTRef, class... Args> void InitializeStorage(Args&&... args) {