Split FlagImpl into 32-byte mutable .data state and a constexpr FlagMeta descriptor. Even after removing redundant pointers, 24 bytes of each 56-byte FlagImpl in writable .data are immutable metadata (name, op, help, and default_value) that never change after compile time, dirtying private .data pages whenever adjacent mutable fields are modified. Factor those immutable fields into a constexpr FlagMeta descriptor and shrink FlagImpl in .data to 32 bytes (meta_, data_guard_, init_control_, file_id_, and mutable state bits). PiperOrigin-RevId: 1006216333
diff --git a/absl/flags/BUILD.bazel b/absl/flags/BUILD.bazel index 59771c7..a43151f 100644 --- a/absl/flags/BUILD.bazel +++ b/absl/flags/BUILD.bazel
@@ -240,6 +240,7 @@ "//absl/base:dynamic_annotations", "//absl/base:fast_type_id", "//absl/base:no_destructor", + "//absl/container:flat_hash_map", "//absl/memory", "//absl/meta:type_traits", "//absl/strings",
diff --git a/absl/flags/CMakeLists.txt b/absl/flags/CMakeLists.txt index 162b654..788ff0a 100644 --- a/absl/flags/CMakeLists.txt +++ b/absl/flags/CMakeLists.txt
@@ -201,6 +201,7 @@ absl::flags_commandlineflag_internal absl::flags_config absl::flags_marshalling + absl::flat_hash_map absl::meta absl::no_destructor absl::synchronization
diff --git a/absl/flags/flag.h b/absl/flags/flag.h index 2e9a9e6..3dade34 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h
@@ -197,11 +197,11 @@ // ABSL_FLAG_IMPL macro definition conditional on ABSL_FLAGS_STRIP_NAMES #define ABSL_FLAG_IMPL_FLAG_PTR(flag) flag -#define ABSL_FLAG_IMPL_HELP_ARG(name) \ - absl::flags_internal::HelpArg<AbslFlagHelpGenFor##name>( \ +#define ABSL_FLAG_IMPL_HELP_ARG(name) \ + ::absl::flags_internal::HelpArg<AbslFlagHelpGenFor##name>( \ FLAGS_help_storage_##name) #define ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name) \ - absl::flags_internal::DefaultArg<Type, AbslFlagDefaultGenFor##name>(0) + ::absl::flags_internal::DefaultArg<Type, AbslFlagDefaultGenFor##name>(0) #if ABSL_FLAGS_STRIP_NAMES #define ABSL_FLAG_IMPL_FLAGNAME(txt) "" @@ -260,22 +260,30 @@ } \ }; +#define ABSL_FLAG_IMPL_DECLARE_META_WRAPPER(name) \ + template <typename AbslFlagType> \ + struct AbslFlagMetaFor##name { \ + static constexpr auto kMeta = ::absl::flags_internal::SelectFlagMeta< \ + AbslFlagType, AbslFlagDefaultGenFor##name, AbslFlagHelpGenFor##name, \ + sizeof(ABSL_FLAG_IMPL_FLAGNAME(#name))>( \ + ABSL_FLAG_IMPL_FLAGNAME(#name), FLAGS_help_storage_##name, 0); \ + }; + // ABSL_FLAG_IMPL // // Note: Name of registrar object is not arbitrary. It is used to "grab" // global name for FLAGS_no<flag_name> symbol, thus preventing the possibility // of defining two flags with names foo and nofoo. -#define ABSL_FLAG_IMPL(Type, name, default_value, help) \ - extern ::absl::Flag<Type> FLAGS_##name; \ - namespace absl /* block flags in namespaces */ {} \ - ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ - ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, help) \ - ABSL_CONST_INIT absl::Flag<Type> FLAGS_##name{ \ - ABSL_FLAG_IMPL_FLAGNAME(#name), ABSL_FLAG_IMPL_TYPENAME(#Type), \ - ABSL_FLAG_IMPL_FILENAME(), ABSL_FLAG_IMPL_HELP_ARG(name), \ - ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name)}; \ - extern absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name; \ - absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name = \ +#define ABSL_FLAG_IMPL(Type, name, default_value, help) \ + extern ::absl::Flag<Type> FLAGS_##name; \ + namespace absl /* block flags in namespaces */ {} \ + ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ + ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, help) \ + ABSL_FLAG_IMPL_DECLARE_META_WRAPPER(name) \ + ABSL_CONST_INIT absl::Flag<Type> FLAGS_##name{ \ + &AbslFlagMetaFor##name<Type>::kMeta.hdr}; \ + extern absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name; \ + absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name = \ ABSL_FLAG_IMPL_REGISTRAR(Type, FLAGS_##name) // ABSL_RETIRED_FLAG
diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc index 5cacf64..3059ce6 100644 --- a/absl/flags/flag_test.cc +++ b/absl/flags/flag_test.cc
@@ -137,6 +137,19 @@ flags::FlagValueStorageKind::kSequenceLocked); EXPECT_EQ(flags::StorageKind<absl::uint128>(), flags::FlagValueStorageKind::kSequenceLocked); + + // These sizes are not hard limits, but we want to keep per-flag size as + // small as possible because production binaries may define thousands of + // flags. + if (sizeof(void*) == 8) { + EXPECT_EQ(sizeof(flags::FlagImpl), 32); + EXPECT_EQ(sizeof(flags::FlagMetaHeader), 1); + EXPECT_EQ(sizeof(flags::FlagMeta), 40); + EXPECT_EQ(sizeof(absl::Flag<bool>), 40); + EXPECT_EQ(sizeof(absl::Flag<int>), 40); + EXPECT_EQ(sizeof(absl::Flag<int64_t>), 40); + EXPECT_EQ(sizeof(absl::Flag<double>), 40); + } } // -------------------------------------------------------------------- @@ -148,17 +161,19 @@ using int128 = absl::int128; using uint128 = absl::uint128; -#define DEFINE_CONSTRUCTED_FLAG(T, dflt, dflt_kind) \ - constexpr flags::FlagDefaultArg f1default##T{ \ - flags::FlagDefaultSrc{dflt}, flags::FlagDefaultKind::dflt_kind}; \ - constexpr absl::Flag<T> f1##T{"f1", #T, "file", help_arg, f1default##T}; \ - ABSL_CONST_INIT absl::Flag<T> f2##T { \ - "f2", #T, "file", \ - {flags::FlagHelpMsg(&TestHelpMsg), flags::FlagHelpKind::kGenFunc}, \ - flags::FlagDefaultArg { \ - flags::FlagDefaultSrc(&TestMakeDflt<T>), \ - flags::FlagDefaultKind::kGenFunc \ - } \ +#define DEFINE_CONSTRUCTED_FLAG(T, dflt, dflt_kind) \ + constexpr flags::FlagDefaultArg f1default##T{ \ + flags::FlagDefaultSrc{dflt}, flags::FlagDefaultKind::dflt_kind}; \ + constexpr flags::FlagMeta f1meta##T = \ + flags::MakeFlagMeta<T>("f1", help_arg, f1default##T); \ + constexpr absl::Flag<T> f1##T{&f1meta##T, help_arg.kind, f1default##T.kind}; \ + constexpr flags::FlagMeta f2meta##T = flags::MakeFlagMeta<T>( \ + "f2", {flags::FlagHelpMsg(&TestHelpMsg), flags::FlagHelpKind::kGenFunc}, \ + flags::FlagDefaultArg{flags::FlagDefaultSrc(&TestMakeDflt<T>), \ + flags::FlagDefaultKind::kGenFunc}); \ + ABSL_CONST_INIT absl::Flag<T> f2##T { \ + &f2meta##T, flags::FlagHelpKind::kGenFunc, \ + flags::FlagDefaultKind::kGenFunc \ } DEFINE_CONSTRUCTED_FLAG(bool, true, kOneWord); @@ -179,9 +194,9 @@ bool TestConstructionFor(const absl::Flag<T>& f1, absl::Flag<T>& f2) { EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Name(), "f1"); EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Help(), "literal help"); - EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Filename(), "file"); + EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Filename(), ""); - flags::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(f2), nullptr) + flags::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(f2), "file") .OnUpdate(TestCallback); EXPECT_EQ(absl::GetFlagReflectionHandle(f2).Name(), "f2"); @@ -1397,3 +1412,48 @@ #endif } // namespace + +ABSL_FLAG(int, concurrent_saver_test_flag, 0, + "test flag for concurrent FlagSaver"); + +namespace { + +TEST_F(FlagTest, ConcurrentFlagSaverAndSetFlag) { + std::atomic<bool> stop{false}; + std::thread writer([&stop]() { + int i = 1; + while (!stop.load(std::memory_order_relaxed)) { + absl::SetFlag(&FLAGS_concurrent_saver_test_flag, i++); + } + }); + for (int iter = 0; iter < 50; ++iter) { + absl::FlagSaver saver; + (void)absl::GetFlag(FLAGS_concurrent_saver_test_flag); + } + stop.store(true, std::memory_order_relaxed); + writer.join(); + EXPECT_GE(absl::GetFlag(FLAGS_concurrent_saver_test_flag), 0); +} + +TEST_F(FlagTest, FilenameInterning) { + constexpr const char* kFileA = "dir/intern_test_a.cc"; + constexpr const char* kFileB = "dir/intern_test_b.cc"; + + absl::Flag<int32_t> f_a1{&f1metaint32_t, help_arg.kind, + f1defaultint32_t.kind}; + absl::Flag<int32_t> f_a2{&f1metaint32_t, help_arg.kind, + f1defaultint32_t.kind}; + absl::Flag<int32_t> f_b{&f1metaint32_t, help_arg.kind, f1defaultint32_t.kind}; + + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_a1), kFileA); + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_a2), kFileA); + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_b), kFileB); + // Duplicate registration preserves the first filename for ODR diagnostics. + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_a1), kFileB); + + EXPECT_EQ(f_a1.Filename(), "dir/intern_test_a.cc"); + EXPECT_EQ(f_a2.Filename(), "dir/intern_test_a.cc"); + EXPECT_EQ(f_b.Filename(), "dir/intern_test_b.cc"); +} + +} // namespace
diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index 662b342..5925240 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc
@@ -23,6 +23,7 @@ #include <array> #include <atomic> #include <cstring> +#include <limits> #include <memory> #include <string> #include <typeinfo> @@ -39,6 +40,7 @@ #include "absl/base/no_destructor.h" #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" #include "absl/flags/config.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/flags/usage_config.h" @@ -104,6 +106,75 @@ s_freelist->push_back(p); } +template <typename T> +struct FlagSideTable { + absl::Mutex mutex{absl::kConstInit}; + absl::flat_hash_map<const FlagImpl*, T>* table ABSL_GUARDED_BY(mutex) = + nullptr; + + T& operator[](const FlagImpl* flag) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { + if (table == nullptr) { + table = new absl::flat_hash_map<const FlagImpl*, T>; + } + return (*table)[flag]; + } + + T Get(const FlagImpl* flag) { + absl::MutexLock l(mutex); + if (table == nullptr) return T{}; + auto it = table->find(flag); + return it != table->end() ? it->second : T{}; + } +}; + +ABSL_CONST_INIT FlagSideTable<FlagCallback*> callbacks; +ABSL_CONST_INIT FlagSideTable<void*> dynamic_defaults; +ABSL_CONST_INIT FlagSideTable<int64_t> overflow_mod_counts; + +FlagCallback* GetOrCreateCallback(const FlagImpl* flag) { + absl::MutexLock l(callbacks.mutex); + FlagCallback*& cb = callbacks[flag]; + if (cb == nullptr) { + cb = new FlagCallback; + } + return cb; +} + +void* GetDynamicDefault(const FlagImpl* flag) { + return dynamic_defaults.Get(flag); +} + +void* SetDynamicDefault(const FlagImpl* flag, void* new_val) { + absl::MutexLock l(dynamic_defaults.mutex); + return std::exchange(dynamic_defaults[flag], new_val); +} + +int64_t GetOverflowModCount(const FlagImpl* flag) { + return overflow_mod_counts.Get(flag); +} + +void IncrementOverflowModCount(const FlagImpl* flag) { + absl::MutexLock l(overflow_mod_counts.mutex); + ++overflow_mod_counts[flag]; +} + +struct FilenameTable { + absl::Mutex mutex; + std::vector<const char*> files ABSL_GUARDED_BY(mutex); +}; + +FilenameTable& GlobalFilenames() { + static absl::NoDestructor<FilenameTable> filenames; + return *filenames; +} + +const char* GetFilenameById(uint16_t id) { + if (id == 0) return ""; + auto& table = GlobalFilenames(); + absl::MutexLock l(table.mutex); + return table.files[id - 1]; +} + } // namespace /////////////////////////////////////////////////////////////////////////////// @@ -129,9 +200,9 @@ on_command_line_(on_command_line), counter_(counter) {} - FlagState(FlagImpl& flag_impl, FlagDefaultSrc default_value) + explicit FlagState(FlagImpl& flag_impl) : flag_impl_(flag_impl), - value_(default_value), + value_(static_cast<int64_t>(0)), modified_(false), on_command_line_(false), counter_(-1) {} @@ -141,7 +212,7 @@ if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kHeapAllocated && flag_impl_.ValueStorageKind() != FlagValueStorageKind::kSequenceLocked) return; - flags_internal::Delete(flag_impl_.op_, value_.heap_allocated); + flags_internal::Delete(flag_impl_.op(), value_.heap_allocated); } private: @@ -161,11 +232,9 @@ union SavedValue { explicit SavedValue(void* v) : heap_allocated(v) {} explicit SavedValue(int64_t v) : one_word(v) {} - explicit SavedValue(FlagDefaultSrc v) : default_value(v) {} void* heap_allocated; int64_t one_word; - FlagDefaultSrc default_value; } value_; bool modified_; bool on_command_line_; @@ -210,25 +279,51 @@ return (reinterpret_cast<mask_t>(ptr_) & mask) != 0; } +void FlagImpl::SetFilename(const char* filename) { + // Keep the first registration's filename so FlagRegistry::RegisterFlag can + // report both files when dynamic symbol interposition merges duplicate flag + // definitions across shared libraries. + if (file_id_ != 0 || filename == nullptr) return; + auto& table = GlobalFilenames(); + absl::MutexLock l(table.mutex); + // All flags in a translation unit register consecutively from one + // .init_array initializer and share one merged .rodata.str1.1 __FILE__ + // string literal, so pointer equality against the last entry deduplicates + // per-TU filenames in O(1) without strcmp or a hash table. + if (!table.files.empty() && table.files.back() == filename) { + file_id_ = static_cast<uint16_t>(table.files.size()); + return; + } + if (ABSL_PREDICT_FALSE(table.files.size() >= + std::numeric_limits<uint16_t>::max())) { + ABSL_INTERNAL_LOG(FATAL, "Too many flag filenames"); + } + table.files.push_back(filename); + file_id_ = static_cast<uint16_t>(table.files.size()); +} + +FlagOpFn FlagImpl::op() const { return PtrMeta()->op; } + void FlagImpl::Init() { new (&data_guard_) absl::Mutex; - auto def_kind = static_cast<FlagDefaultKind>(def_kind_); + FlagDefaultKind def_kind = + mu_state_.load(std::memory_order_relaxed).DefaultKind(); switch (ValueStorageKind()) { case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { alignas(int64_t) std::array<char, sizeof(int64_t)> buf{}; if (def_kind == FlagDefaultKind::kGenFunc) { - (*default_value_.gen_func)(buf.data()); + (*PtrMeta()->default_value.gen_func)(buf.data()); } else { - assert(def_kind != FlagDefaultKind::kDynamicValue); - std::memcpy(buf.data(), &default_value_, Sizeof(op_)); + assert(def_kind == FlagDefaultKind::kOneWord); + std::memcpy(buf.data(), &PtrMeta()->default_value, Sizeof(op())); } if (ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit) { // We presume here the memory layout of FlagValueAndInitBit struct. uint8_t initialized = 1; - std::memcpy(buf.data() + Sizeof(op_), &initialized, + std::memcpy(buf.data() + Sizeof(op()), &initialized, sizeof(initialized)); } // Type can contain valid uninitialized bits, e.g. padding. @@ -238,38 +333,44 @@ break; } case FlagValueStorageKind::kSequenceLocked: { - // For this storage kind the default_value_ always points to gen_func - // during initialization. - assert(def_kind == FlagDefaultKind::kGenFunc); - (*default_value_.gen_func)(AtomicBufferValue()); + if (def_kind == FlagDefaultKind::kGenFunc) { + (*PtrMeta()->default_value.gen_func)(AtomicBufferValue()); + } else { + assert(def_kind == FlagDefaultKind::kOneWord); + std::memcpy(static_cast<void*>(AtomicBufferValue()), + &PtrMeta()->default_value, Sizeof(op())); + } + SeqLock().MarkInitialized(); break; } - case FlagValueStorageKind::kHeapAllocated: - // For this storage kind the default_value_ always points to gen_func - // during initialization. - assert(def_kind == FlagDefaultKind::kGenFunc); + case FlagValueStorageKind::kHeapAllocated: { // Flag value initially points to the internal buffer. MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire); - (*default_value_.gen_func)(ptr_value.Ptr()); + if (def_kind == FlagDefaultKind::kGenFunc) { + (*PtrMeta()->default_value.gen_func)(ptr_value.Ptr()); + } else { + assert(def_kind == FlagDefaultKind::kOneWord); + std::memcpy(ptr_value.Ptr(), &PtrMeta()->default_value, Sizeof(op())); + } // Default value is a candidate for an unprotected read. PtrStorage().store(MaskedPointer(ptr_value.Ptr(), true), std::memory_order_release); break; + } } - seq_lock_.MarkInitialized(); } -absl::Mutex& FlagImpl::DataGuard() const { +absl::Mutex* FlagImpl::DataGuard() const { absl::call_once(const_cast<FlagImpl*>(this)->init_control_, &FlagImpl::Init, const_cast<FlagImpl*>(this)); // data_guard_ is initialized inside Init. - return *reinterpret_cast<absl::Mutex*>(&data_guard_); + return reinterpret_cast<absl::Mutex*>(data_guard_); } void FlagImpl::AssertValidType(FlagFastTypeId rhs_type_id, const std::type_info* (*gen_rtti)()) const { - FlagFastTypeId lhs_type_id = flags_internal::FastTypeId(op_); + FlagFastTypeId lhs_type_id = flags_internal::FastTypeId(op()); // `rhs_type_id` is the fast type id corresponding to the declaration // visible at the call site. `lhs_type_id` is the fast type id @@ -278,7 +379,7 @@ if (ABSL_PREDICT_TRUE(lhs_type_id == rhs_type_id)) return; const std::type_info* lhs_runtime_type_id = - flags_internal::RuntimeTypeId(op_); + flags_internal::RuntimeTypeId(op()); const std::type_info* rhs_runtime_type_id = (*gen_rtti)(); if (lhs_runtime_type_id == rhs_runtime_type_id) return; @@ -294,19 +395,31 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const { void* res = nullptr; - switch (DefaultKind()) { + switch (mu_state_.load(std::memory_order_relaxed).DefaultKind()) { case FlagDefaultKind::kDynamicValue: - res = flags_internal::Clone(op_, default_value_.dynamic_value); + res = flags_internal::Clone(op(), GetDynamicDefault(this)); break; case FlagDefaultKind::kGenFunc: - res = flags_internal::Alloc(op_); - (*default_value_.gen_func)(res); + res = flags_internal::Alloc(op()); + (*PtrMeta()->default_value.gen_func)(res); break; - default: - res = flags_internal::Clone(op_, &default_value_); + case FlagDefaultKind::kOneWord: + res = flags_internal::Alloc(op()); + std::memcpy(res, &PtrMeta()->default_value, Sizeof(op())); break; } - return {res, DynValueDeleter{op_}}; + return {res, DynValueDeleter{op()}}; +} + +void FlagImpl::RecordMutation() { + MuState s = mu_state_.load(std::memory_order_relaxed); + if (s.mod_count < kMaxInlineModCount) { + ++s.mod_count; + } else { + IncrementOverflowModCount(this); + } + s.modified = true; + mu_state_.store(s, std::memory_order_release); } void FlagImpl::StoreValue(const void* src, ValueSource source) { @@ -315,13 +428,12 @@ case FlagValueStorageKind::kOneWordAtomic: { // Load the current value to avoid setting 'init' bit manually. int64_t one_word_val = OneWordValue().load(std::memory_order_acquire); - std::memcpy(&one_word_val, src, Sizeof(op_)); + std::memcpy(&one_word_val, src, Sizeof(op())); OneWordValue().store(one_word_val, std::memory_order_release); - seq_lock_.IncrementModificationCount(); break; } case FlagValueStorageKind::kSequenceLocked: { - seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_)); + SeqLock().Write(AtomicBufferValue(), src, Sizeof(op())); break; } case FlagValueStorageKind::kHeapAllocated: @@ -335,76 +447,84 @@ // instead we move the memory to a freelist so it can still be safely // accessed, and allocate a new one for the new value. AddToFreelist(ptr_value.Ptr()); - ptr_value = MaskedPointer(Clone(op_, src), source == kCommandLine); + ptr_value = MaskedPointer(Clone(op(), src), source == kCommandLine); } else { // Current value either was set programmatically or was never read. // We can reuse the memory since all accesses to this value (if any) // were protected by mutex. That said, if a new value comes from command // line it now becomes a candidate for an unprotected read. - ptr_value.Set(op_, src, source == kCommandLine); + ptr_value.Set(op(), src, source == kCommandLine); } PtrStorage().store(ptr_value, std::memory_order_release); - seq_lock_.IncrementModificationCount(); break; } - modified_ = true; + RecordMutation(); InvokeCallback(); } -absl::string_view FlagImpl::Name() const { return name_; } +absl::string_view FlagImpl::Name() const { return PtrMeta()->name; } -absl::string_view FlagImpl::TypeName() const { return type_name_; } +absl::string_view FlagImpl::TypeName() const { + return flags_internal::TypeName(op()); +} std::string FlagImpl::Filename() const { - return flags_internal::GetUsageConfig().normalize_filename(filename_); + return flags_internal::GetUsageConfig().normalize_filename( + GetFilenameById(file_id_)); } std::string FlagImpl::Help() const { - return HelpSourceKind() == FlagHelpKind::kLiteral ? help_.literal - : help_.gen_func(); + return HelpSourceKind() == FlagHelpKind::kLiteral + ? PtrMeta()->help.literal + : PtrMeta()->help.gen_func(); } FlagFastTypeId FlagImpl::TypeId() const { - return flags_internal::FastTypeId(op_); + return flags_internal::FastTypeId(op()); } int64_t FlagImpl::ModificationCount() const { - return seq_lock_.ModificationCount(); + const uint8_t mod_count = + mu_state_.load(std::memory_order_relaxed).mod_count; + if (ABSL_PREDICT_TRUE(mod_count < kMaxInlineModCount)) { + return mod_count; + } + return kMaxInlineModCount + GetOverflowModCount(this); } bool FlagImpl::IsSpecifiedOnCommandLine() const { - absl::MutexLock l(DataGuard()); - return on_command_line_; + absl::MutexLock l(*DataGuard()); + return mu_state_.load(std::memory_order_relaxed).on_command_line; } std::string FlagImpl::DefaultValue() const { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); auto obj = MakeInitValue(); - return flags_internal::Unparse(op_, obj.get()); + return flags_internal::Unparse(op(), obj.get()); } std::string FlagImpl::CurrentValue() const { - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized switch (ValueStorageKind()) { case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { const auto one_word_val = absl::bit_cast<std::array<char, sizeof(int64_t)>>( OneWordValue().load(std::memory_order_acquire)); - return flags_internal::Unparse(op_, one_word_val.data()); + return flags_internal::Unparse(op(), one_word_val.data()); } case FlagValueStorageKind::kSequenceLocked: { - std::unique_ptr<void, DynValueDeleter> cloned(flags_internal::Alloc(op_), - DynValueDeleter{op_}); + std::unique_ptr<void, DynValueDeleter> cloned(flags_internal::Alloc(op()), + DynValueDeleter{op()}); ReadSequenceLockedData(cloned.get()); - return flags_internal::Unparse(op_, cloned.get()); + return flags_internal::Unparse(op(), cloned.get()); } case FlagValueStorageKind::kHeapAllocated: { - absl::MutexLock l(guard); + absl::MutexLock l(*guard); return flags_internal::Unparse( - op_, PtrStorage().load(std::memory_order_acquire).Ptr()); + op(), PtrStorage().load(std::memory_order_acquire).Ptr()); } } @@ -412,22 +532,24 @@ } void FlagImpl::SetCallback(const FlagCallbackFunc mutation_callback) { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); - if (callback_ == nullptr) { - callback_ = new FlagCallback; - } - callback_->func = mutation_callback; + FlagCallback* cb = GetOrCreateCallback(this); + cb->func = mutation_callback; + MuState s = mu_state_.load(std::memory_order_relaxed); + s.has_callback = true; + mu_state_.store(s, std::memory_order_release); InvokeCallback(); } void FlagImpl::InvokeCallback() const { - if (!callback_) return; + if (!mu_state_.load(std::memory_order_relaxed).has_callback) return; + FlagCallback* cb_entry = GetOrCreateCallback(this); // Make a copy of the C-style function pointer that we are about to invoke // before we release the lock guarding it. - FlagCallbackFunc cb = callback_->func; + FlagCallbackFunc cb = cb_entry->func; // If the flag has a mutation callback this function invokes it. While the // callback is being invoked the primary flag's mutex is unlocked and it is @@ -440,44 +562,38 @@ // and it also can be different by the time the callback invocation is // completed. Requires that *primary_lock be held in exclusive mode; it may be // released and reacquired by the implementation. - MutexRelock relock(DataGuard()); - absl::MutexLock lock(callback_->guard); + MutexRelock relock(*DataGuard()); + absl::MutexLock lock(cb_entry->guard); cb(); } -// Returns true if an unmodified absl::Flag (i.e. `seq_lock_.IsUnmodified()`) -// can skip lazy initialization and state allocation in `SaveState()`, and -// instead reconstructs from the provided default value if modified before -// restoration. bool FlagImpl::CanSkipInitOnSave() const { - if (!seq_lock_.IsUnmodified()) return false; + const MuState state = mu_state_.load(std::memory_order_acquire); + if (state.mod_count != 0) return false; + const FlagDefaultKind def_kind = state.DefaultKind(); + if (def_kind == FlagDefaultKind::kDynamicValue) return false; switch (ValueStorageKind()) { case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: - // Only `kOneWord` defaults are inlinable compile-time constants; other - // kinds may have dynamic defaults or side effects. - return static_cast<FlagDefaultKind>(def_kind_) == - FlagDefaultKind::kOneWord; + return def_kind != FlagDefaultKind::kGenFunc; case FlagValueStorageKind::kSequenceLocked: - // Always uses a default-generator function when unmodified. return true; case FlagValueStorageKind::kHeapAllocated: - // Built-in types have side-effect-free defaults; user-defined types may - // have custom construction or validation side effects. - return !ShouldValidateFlagValue(flags_internal::FastTypeId(op_)); + return !ShouldValidateFlagValue(flags_internal::FastTypeId(op())); } return false; } std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { if (ABSL_PREDICT_TRUE(CanSkipInitOnSave())) { - return std::make_unique<FlagState>(*this, default_value_); + return std::make_unique<FlagState>(*this); } - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); - bool modified = modified_; - bool on_command_line = on_command_line_; + const MuState state = mu_state_.load(std::memory_order_relaxed); + bool modified = state.modified; + bool on_command_line = state.on_command_line; switch (ValueStorageKind()) { case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { @@ -486,10 +602,10 @@ on_command_line, ModificationCount()); } case FlagValueStorageKind::kSequenceLocked: { - void* cloned = flags_internal::Alloc(op_); + void* cloned = flags_internal::Alloc(op()); // Read is guaranteed to be successful because we hold the lock. bool success = - seq_lock_.TryRead(cloned, AtomicBufferValue(), Sizeof(op_)); + SeqLock().TryRead(cloned, AtomicBufferValue(), Sizeof(op())); assert(success); static_cast<void>(success); return std::make_unique<FlagState>(*this, cloned, modified, @@ -499,7 +615,7 @@ return std::make_unique<FlagState>( *this, flags_internal::Clone( - op_, PtrStorage().load(std::memory_order_acquire).Ptr()), + op(), PtrStorage().load(std::memory_order_acquire).Ptr()), modified, on_command_line, ModificationCount()); } } @@ -507,39 +623,35 @@ } bool FlagImpl::RestoreState(const FlagState& flag_state) { - if (ABSL_PREDICT_TRUE(flag_state.counter_ < 0 && seq_lock_.IsUnmodified())) { + if (ABSL_PREDICT_TRUE(flag_state.counter_ < 0 && IsUnmodifiedFast())) { return false; } - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); if (flag_state.counter_ == ModificationCount()) { return false; } - switch (ValueStorageKind()) { - case FlagValueStorageKind::kValueAndInitBit: - case FlagValueStorageKind::kOneWordAtomic: - StoreValue( - flag_state.counter_ < 0 - ? static_cast<const void*>(&flag_state.value_.default_value) - : static_cast<const void*>(&flag_state.value_.one_word), - kProgrammaticChange); - break; - case FlagValueStorageKind::kSequenceLocked: - case FlagValueStorageKind::kHeapAllocated: - if (flag_state.counter_ < 0) { - std::unique_ptr<void, DynValueDeleter> init_value( - flags_internal::Alloc(op_), DynValueDeleter{op_}); - (*flag_state.value_.default_value.gen_func)(init_value.get()); - StoreValue(init_value.get(), kProgrammaticChange); - } else { + if (flag_state.counter_ < 0) { + auto init_value = MakeInitValue(); + StoreValue(init_value.get(), kProgrammaticChange); + } else { + switch (ValueStorageKind()) { + case FlagValueStorageKind::kValueAndInitBit: + case FlagValueStorageKind::kOneWordAtomic: + StoreValue(&flag_state.value_.one_word, kProgrammaticChange); + break; + case FlagValueStorageKind::kSequenceLocked: + case FlagValueStorageKind::kHeapAllocated: StoreValue(flag_state.value_.heap_allocated, kProgrammaticChange); - } - break; + break; + } } - modified_ = flag_state.modified_; - on_command_line_ = flag_state.on_command_line_; + MuState s = mu_state_.load(std::memory_order_relaxed); + s.modified = flag_state.modified_; + s.on_command_line = flag_state.on_command_line_; + mu_state_.store(s, std::memory_order_release); return true; } @@ -548,7 +660,7 @@ StorageT* FlagImpl::OffsetValue() const { char* p = reinterpret_cast<char*>(const_cast<FlagImpl*>(this)); // The offset is deduced via Flag value type specific op_. - ptrdiff_t offset = flags_internal::ValueOffset(op_); + ptrdiff_t offset = flags_internal::ValueOffset(op()); return reinterpret_cast<StorageT*>(p + offset); } @@ -558,6 +670,14 @@ return OffsetValue<std::atomic<uint64_t>>(); } +flags_internal::SequenceLock& FlagImpl::SeqLock() const { + assert(ValueStorageKind() == FlagValueStorageKind::kSequenceLocked); + size_t num_words = flags_internal::AlignUp(Sizeof(op()), sizeof(uint64_t)) / + sizeof(uint64_t); + return *reinterpret_cast<flags_internal::SequenceLock*>(AtomicBufferValue() + + num_words); +} + std::atomic<int64_t>& FlagImpl::OneWordValue() const { assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic || ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); @@ -578,7 +698,7 @@ std::unique_ptr<void, DynValueDeleter> tentative_value = MakeInitValue(); std::string parse_err; - if (!flags_internal::Parse(op_, value, tentative_value.get(), &parse_err)) { + if (!flags_internal::Parse(op(), value, tentative_value.get(), &parse_err)) { absl::string_view err_sep = parse_err.empty() ? "" : "; "; err = absl::StrCat("Illegal value '", value, "' specified for flag '", Name(), "'", err_sep, parse_err); @@ -589,13 +709,13 @@ } void FlagImpl::Read(void* dst) const { - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized switch (ValueStorageKind()) { case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { const int64_t one_word_val = OneWordValue().load(std::memory_order_acquire); - std::memcpy(dst, &one_word_val, Sizeof(op_)); + std::memcpy(dst, &one_word_val, Sizeof(op())); break; } case FlagValueStorageKind::kSequenceLocked: { @@ -603,10 +723,10 @@ break; } case FlagValueStorageKind::kHeapAllocated: { - absl::MutexLock l(guard); + absl::MutexLock l(*guard); MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire); - flags_internal::CopyConstruct(op_, ptr_value.Ptr(), dst); + flags_internal::CopyConstruct(op(), ptr_value.Ptr(), dst); // For unprotected read candidates, mark that the value as has been read. if (ptr_value.IsUnprotectedReadCandidate() && !ptr_value.HasBeenRead()) { @@ -621,14 +741,14 @@ int64_t FlagImpl::ReadOneWord() const { assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic || ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized (void)guard; return OneWordValue().load(std::memory_order_acquire); } bool FlagImpl::ReadOneBool() const { assert(ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized (void)guard; return absl::bit_cast<FlagValueAndInitBit<bool>>( OneWordValue().load(std::memory_order_acquire)) @@ -636,28 +756,28 @@ } void FlagImpl::ReadSequenceLockedData(void* dst) const { - size_t size = Sizeof(op_); + size_t size = Sizeof(op()); // Attempt to read using the sequence lock. - if (ABSL_PREDICT_TRUE(seq_lock_.TryRead(dst, AtomicBufferValue(), size))) { + if (ABSL_PREDICT_TRUE(SeqLock().TryRead(dst, AtomicBufferValue(), size))) { return; } // We failed due to contention. Acquire the lock to prevent contention // and try again. - absl::ReaderMutexLock l(DataGuard()); - bool success = seq_lock_.TryRead(dst, AtomicBufferValue(), size); + absl::ReaderMutexLock l(*DataGuard()); + bool success = SeqLock().TryRead(dst, AtomicBufferValue(), size); assert(success); static_cast<void>(success); } void FlagImpl::Write(const void* src) { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); - if (ShouldValidateFlagValue(flags_internal::FastTypeId(op_))) { - std::unique_ptr<void, DynValueDeleter> obj{flags_internal::Clone(op_, src), - DynValueDeleter{op_}}; + if (ShouldValidateFlagValue(flags_internal::FastTypeId(op()))) { + std::unique_ptr<void, DynValueDeleter> obj{flags_internal::Clone(op(), src), + DynValueDeleter{op()}}; std::string ignored_error; - std::string src_as_str = flags_internal::Unparse(op_, src); - if (!flags_internal::Parse(op_, src_as_str, obj.get(), &ignored_error)) { + std::string src_as_str = flags_internal::Unparse(op(), src); + if (!flags_internal::Parse(op(), src_as_str, obj.get(), &ignored_error)) { ABSL_INTERNAL_LOG(ERROR, absl::StrCat("Attempt to set flag '", Name(), "' to invalid value ", src_as_str)); } @@ -676,7 +796,7 @@ // The mode is selected based on 'set_mode' parameter. bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode, ValueSource source, std::string& err) { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); switch (set_mode) { case SET_FLAGS_VALUE: { @@ -687,13 +807,15 @@ StoreValue(tentative_value.get(), source); if (source == kCommandLine) { - on_command_line_ = true; + MuState s = mu_state_.load(std::memory_order_relaxed); + s.on_command_line = true; + mu_state_.store(s, std::memory_order_release); } break; } case SET_FLAG_IF_DEFAULT: { // set the flag's value, but only if it hasn't been set by someone else - if (modified_) { + if (mu_state_.load(std::memory_order_relaxed).modified) { // TODO(rogeeff): review and fix this semantic. Currently we do not fail // in this case if flag is modified. This is misleading since the flag's // value is not updated even though we return true. @@ -712,19 +834,23 @@ auto tentative_value = TryParse(value, err); if (!tentative_value) return false; - if (DefaultKind() == FlagDefaultKind::kDynamicValue) { - void* old_value = default_value_.dynamic_value; - default_value_.dynamic_value = tentative_value.release(); + void* new_default = tentative_value.release(); + void* old_value = SetDynamicDefault(this, new_default); + MuState s = mu_state_.load(std::memory_order_relaxed); + const bool was_modified = s.modified; + if (s.DefaultKind() == FlagDefaultKind::kDynamicValue) { tentative_value.reset(old_value); } else { - default_value_.dynamic_value = tentative_value.release(); - def_kind_ = static_cast<uint8_t>(FlagDefaultKind::kDynamicValue); + s.def_kind = static_cast<uint8_t>(FlagDefaultKind::kDynamicValue); + mu_state_.store(s, std::memory_order_release); } - if (!modified_) { + if (!was_modified) { // Need to set both default value *and* current, in this case. - StoreValue(default_value_.dynamic_value, source); - modified_ = false; + StoreValue(new_default, source); + s = mu_state_.load(std::memory_order_relaxed); + s.modified = false; + mu_state_.store(s, std::memory_order_release); } break; } @@ -736,11 +862,11 @@ void FlagImpl::CheckDefaultValueParsingRoundtrip() const { std::string v = DefaultValue(); - absl::MutexLock lock(DataGuard()); + absl::MutexLock lock(*DataGuard()); auto dst = MakeInitValue(); std::string error; - if (!flags_internal::Parse(op_, v, dst.get(), &error)) { + if (!flags_internal::Parse(op(), v, dst.get(), &error)) { ABSL_INTERNAL_LOG( FATAL, absl::StrCat("Flag ", Name(), " (from ", Filename(), @@ -753,11 +879,11 @@ } bool FlagImpl::ValidateInputValue(absl::string_view value) const { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); auto obj = MakeInitValue(); std::string ignored_error; - return flags_internal::Parse(op_, value, obj.get(), &ignored_error); + return flags_internal::Parse(op(), value, obj.get(), &ignored_error); } } // namespace flags_internal
diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index 8baff43..6b70425 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h
@@ -44,9 +44,12 @@ #include "absl/synchronization/mutex.h" #include "absl/utility/utility.h" - namespace absl { +namespace absl { ABSL_NAMESPACE_BEGIN +class Duration; +class Time; + /////////////////////////////////////////////////////////////////////////////// // Forward declaration of absl::Flag<T> public API. namespace flags_internal { @@ -86,6 +89,7 @@ kParse, kUnparse, kValueOffset, + kTypeName, }; using FlagOpFn = void* (*)(FlagOp, const void*, void*, void*); @@ -154,6 +158,29 @@ return static_cast<ptrdiff_t>(reinterpret_cast<intptr_t>( op(FlagOp::kValueOffset, nullptr, nullptr, nullptr))); } +// Returns string representation of the flag's value type. +inline absl::string_view TypeName(FlagOpFn op) { + absl::string_view result; + op(FlagOp::kTypeName, nullptr, &result, nullptr); + return result; +} + +template <typename T> +constexpr absl::string_view GenTypeName() { +#if ABSL_FLAGS_STRIP_NAMES + return ""; +#elif defined(__clang__) || defined(__GNUC__) + absl::string_view name = __PRETTY_FUNCTION__; + constexpr absl::string_view kKey = "T = "; + const size_t start = name.find(kKey); + if (start == absl::string_view::npos) return ""; + name.remove_prefix(start + kKey.size()); + const size_t end = name.find_first_of(";]"); + return end == absl::string_view::npos ? name : name.substr(0, end); +#else + return ""; +#endif +} // Returns an address of RTTI's typeid(T). template <typename T> @@ -260,7 +287,7 @@ enum class FlagDefaultKind : uint8_t { kDynamicValue = 0, kGenFunc = 1, - kOneWord = 2 // for default values UP to one word in size + kOneWord = 2, // inline default value in descriptor }; struct FlagDefaultArg { @@ -437,7 +464,7 @@ template <typename T> struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue { constexpr FlagValue() : FlagOneWordValue(0) {} - bool Get(const SequenceLock&, T& dst) const { + bool Get(T& dst) const { int64_t storage = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(storage == 0)) { // This assert is to ensure that the initialization inside FlagImpl::Init @@ -458,7 +485,7 @@ template <typename T> struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { constexpr FlagValue() : FlagOneWordValue() {} - bool Get(const SequenceLock&, T& dst) const { + bool Get(T& dst) const { int64_t one_word_val = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(one_word_val == FlagOneWordValue::Uninitialized())) { return false; @@ -476,8 +503,17 @@ // value once it is initialized. template <typename T> struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> { - bool Get(const SequenceLock& lock, T& dst) const { - return lock.TryRead(&dst, value_words, sizeof(T)); + constexpr FlagValue() : FlagValue(std::make_index_sequence<kNumWords>{}) {} + + bool Get(T& dst) const { + // FlagImpl::SeqLock() computes `&seq_lock` via `AtomicBufferValue() + + // kNumWords`, so verify no padding or fields sit between `value_words` + // and `seq_lock`. + static_assert( + offsetof(FlagValue, seq_lock) - offsetof(FlagValue, value_words) == + kNumWords * sizeof(std::atomic<uint64_t>), + "Unexpected memory layout of sequence-locked FlagValue"); + return seq_lock.TryRead(&dst, value_words, sizeof(T)); } static constexpr int kNumWords = @@ -485,6 +521,12 @@ alignas(T) alignas( std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords]; + flags_internal::SequenceLock seq_lock; + + private: + template <size_t... Is> + constexpr explicit FlagValue(std::index_sequence<Is...>) + : value_words{(static_cast<void>(Is), uint64_t{0})...} {} }; // This specialization represents the storage of flag values types with the @@ -531,7 +573,7 @@ // ensure Init is done before any access to the flag value. constexpr FlagValue() : FlagMaskedPointerValue(&buffer[0]) {} - bool Get(const SequenceLock&, T& dst) const { + bool Get(T& dst) const { MaskedPointer ptr_value = value.load(std::memory_order_acquire); if (ABSL_PREDICT_TRUE(ptr_value.AllowsUnprotectedRead())) { @@ -569,6 +611,56 @@ FlagOpFn op; }; +struct FlagMetaHeader { + constexpr static uint8_t PackKinds(FlagValueStorageKind vk, FlagHelpKind hk, + FlagDefaultKind dk) { + return static_cast<uint8_t>((static_cast<uint8_t>(vk) & 0x7u) | + ((static_cast<uint8_t>(hk) & 0x1u) << 3) | + ((static_cast<uint8_t>(dk) & 0x3u) << 4)); + } + + uint8_t kinds; +}; + +// Pointer-based descriptor for flag metadata (40 B). +struct FlagMeta { + constexpr FlagMeta(FlagMetaHeader hdr_arg, const char* name_arg, + FlagHelpMsg help_arg, FlagDefaultSrc default_value_arg, + FlagOpFn op_arg) + : hdr(hdr_arg), + name(name_arg), + help(help_arg), + default_value(default_value_arg), + op(op_arg) {} + + FlagMetaHeader hdr; + // Flag's name passed to ABSL_FLAG as second arg. + const char* const name; + // Help message literal or function to generate it. + const FlagHelpMsg help; + // Initial default value source specified in ABSL_FLAG. + const FlagDefaultSrc default_value; + // Type-specific operations vtable. + const FlagOpFn op; +}; + +template <typename T> +constexpr FlagMeta MakeFlagMeta(const char* name, FlagHelpArg help, + FlagDefaultArg default_arg) { + return FlagMeta( + FlagMetaHeader{FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(), + help.kind, default_arg.kind)}, + name, help.source, default_arg.source, &FlagOps<T>); +} + +template <typename ValueT, typename GenT, typename HelpGenT, size_t NameLen, + typename HelpStorageT> +constexpr FlagMeta SelectFlagMeta(const char* name, + const HelpStorageT& help_storage, int) { + return MakeFlagMeta<ValueT>(name, HelpArg<HelpGenT>(help_storage), + DefaultArg<ValueT, GenT>(0)); +} + class FlagState; // These are only used as constexpr global objects. @@ -579,37 +671,55 @@ #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #endif class FlagImpl final : public CommandLineFlag { + private: + static constexpr uint8_t kMaxInlineModCount = 7; + + struct MuState { + uint8_t def_kind : 2; + bool modified : 1; + bool on_command_line : 1; + bool has_callback : 1; + uint8_t mod_count : 3; + + constexpr FlagDefaultKind DefaultKind() const { + return static_cast<FlagDefaultKind>(def_kind); + } + }; + static_assert(sizeof(MuState) == 1, "MuState must be 1 byte"); + static_assert(std::atomic<MuState>::is_always_lock_free, + "std::atomic<MuState> must be lock-free"); + public: - constexpr FlagImpl(const char* name, const char* type_name, - const char* filename, FlagOpFn op, FlagHelpArg help, - FlagValueStorageKind value_kind, - FlagDefaultArg default_arg) - : name_(name), - type_name_(type_name), - filename_(filename), - op_(op), - help_(help.source), - help_source_kind_(static_cast<uint8_t>(help.kind)), - value_storage_kind_(static_cast<uint8_t>(value_kind)), - def_kind_(static_cast<uint8_t>(default_arg.kind)), - modified_(false), - on_command_line_(false), - callback_(nullptr), - default_value_(default_arg.source), - data_guard_{} {} + constexpr explicit FlagImpl(const FlagMetaHeader* meta) + : meta_(meta), + init_control_{}, + file_id_(0), + mu_state_(MuState{static_cast<uint8_t>((meta->kinds >> 4) & 0x3u), + false, false, false, 0}), + reserved_(0) {} + + constexpr FlagImpl(const FlagMeta* meta, FlagHelpKind /*help_kind*/, + FlagValueStorageKind /*value_kind*/, + FlagDefaultKind default_kind) + : meta_(&meta->hdr), + init_control_{}, + file_id_(0), + mu_state_(MuState{static_cast<uint8_t>(default_kind), false, false, + false, 0}), + reserved_(0) {} // Constant access methods - int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(DataGuard()); - bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(DataGuard()); - void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(DataGuard()); - void Read(bool* value) const ABSL_LOCKS_EXCLUDED(DataGuard()) { + int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(*DataGuard()); + bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(*DataGuard()); + void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + void Read(bool* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { *value = ReadOneBool(); } template <typename T, std::enable_if_t<flags_internal::StorageKind<T>() == FlagValueStorageKind::kOneWordAtomic, int> = 0> - void Read(T* value) const ABSL_LOCKS_EXCLUDED(DataGuard()) { + void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { int64_t v = ReadOneWord(); std::memcpy(value, static_cast<const void*>(&v), sizeof(T)); } @@ -617,17 +727,18 @@ std::enable_if_t<flags_internal::StorageKind<T>() == FlagValueStorageKind::kValueAndInitBit, int> = 0> - void Read(T* value) const ABSL_LOCKS_EXCLUDED(DataGuard()) { + void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { *value = absl::bit_cast<FlagValueAndInitBit<T>>(ReadOneWord()).value; } // Mutating access methods - void Write(const void* src) ABSL_LOCKS_EXCLUDED(DataGuard()); + void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard()); + void SetFilename(const char* filename); // Interfaces to operate on callbacks. void SetCallback(const FlagCallbackFunc mutation_callback) - ABSL_LOCKS_EXCLUDED(DataGuard()); - void InvokeCallback() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); + void InvokeCallback() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Used in read/write operations to validate source/target has correct type. // For example if flag is declared as absl::Flag<int> FLAGS_foo, a call to @@ -643,12 +754,17 @@ friend class Flag; friend class FlagState; + FlagOpFn op() const; + const FlagMeta* PtrMeta() const { + return reinterpret_cast<const FlagMeta*>(meta_); + } + // Ensures that `data_guard_` is initialized and returns it. - absl::Mutex& DataGuard() const + absl::Mutex* DataGuard() const ABSL_LOCK_RETURNED(reinterpret_cast<absl::Mutex*>(data_guard_)); // Returns heap allocated value of type T initialized with default value. std::unique_ptr<void, DynValueDeleter> MakeInitValue() const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Flag initialization called via absl::call_once. void Init(); @@ -656,7 +772,7 @@ // respect const correctness, so be very careful using them. // This is a shared helper routine which encapsulates most of the magic. Since - // it is only used inside the three routines below, which are defined in + // it is only used inside the routines below, which are defined in // flag.cc, we can define it in that file as well. template <typename StorageT> StorageT* OffsetValue() const; @@ -664,6 +780,9 @@ // The same as above, but used for sequencelock-protected storage. std::atomic<uint64_t>* AtomicBufferValue() const; + // Accessor for the SequenceLock inside FlagValue<T, kSequenceLocked>. + flags_internal::SequenceLock& SeqLock() const; + // This is an accessor for a value stored as one word atomic. Returns a // mutable reference to an atomic value. std::atomic<int64_t>& OneWordValue() const; @@ -674,25 +793,22 @@ // returns new value. Otherwise returns nullptr. std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value, std::string& err) const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Stores the flag value based on the pointer to the source. void StoreValue(const void* src, ValueSource source) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); - // Copy the flag data, protected by `seq_lock_` into `dst`. + // Copy the flag data, protected by `SeqLock()` into `dst`. // // REQUIRES: ValueStorageKind() == kSequenceLocked. - void ReadSequenceLockedData(void* dst) const ABSL_LOCKS_EXCLUDED(DataGuard()); + void ReadSequenceLockedData(void* dst) const + ABSL_LOCKS_EXCLUDED(*DataGuard()); FlagHelpKind HelpSourceKind() const { - return static_cast<FlagHelpKind>(help_source_kind_); + return static_cast<FlagHelpKind>((meta_->kinds >> 3) & 0x1u); } FlagValueStorageKind ValueStorageKind() const { - return static_cast<FlagValueStorageKind>(value_storage_kind_); - } - FlagDefaultKind DefaultKind() const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()) { - return static_cast<FlagDefaultKind>(def_kind_); + return static_cast<FlagValueStorageKind>(meta_->kinds & 0x7u); } // CommandLineFlag interface implementation @@ -702,86 +818,63 @@ std::string Help() const override; FlagFastTypeId TypeId() const override; bool IsSpecifiedOnCommandLine() const override - ABSL_LOCKS_EXCLUDED(DataGuard()); - std::string DefaultValue() const override ABSL_LOCKS_EXCLUDED(DataGuard()); - std::string CurrentValue() const override ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); + std::string DefaultValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + std::string CurrentValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard()); bool ValidateInputValue(absl::string_view value) const override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); void CheckDefaultValueParsingRoundtrip() const override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); - int64_t ModificationCount() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + int64_t ModificationCount() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + // Lock-free check used by SaveState() and RestoreState() to skip lazy + // initialization on flags that have never been mutated. Safe without + // DataGuard() because all mutations update mu_state_ under DataGuard() + // with release semantics after writing the flag value, and observing a + // zero modification count means the flag still holds its initial + // compile-time state. + bool IsUnmodifiedFast() const { + return mu_state_.load(std::memory_order_acquire).mod_count == 0; + } bool CanSkipInitOnSave() const; + void RecordMutation() ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Interfaces to save and restore flags to/from persistent state. // Returns current flag state or nullptr if flag does not support // saving and restoring a state. std::unique_ptr<FlagStateInterface> SaveState() override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); // Restores the flag state to the supplied state object. If there is // nothing to restore returns false. Otherwise returns true. bool RestoreState(const FlagState& flag_state) - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); bool ParseFrom(absl::string_view value, FlagSettingMode set_mode, ValueSource source, std::string& error) override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); - // Immutable flag's state. + // Pointer to immutable flag descriptor in .rodata / .data.rel.ro. + const FlagMetaHeader* const meta_; - // Flags name passed to ABSL_FLAG as second arg. - const char* const name_; - - // Flags type passed to ABSL_FLAG as first arg. - const char* const type_name_; - - // The file name where ABSL_FLAG resides. - const char* const filename_; - // Type-specific operations vtable. - const FlagOpFn op_; - // Help message literal or function to generate it. - const FlagHelpMsg help_; - // Indicates if help message was supplied as literal or generator func. - const uint8_t help_source_kind_ : 1; - // Kind of storage this flag is using for the flag's value. - const uint8_t value_storage_kind_ : 2; - - uint8_t : 0; // The bytes containing the const bitfields must not be - // shared with bytes containing the mutable bitfields. - - // Mutable flag's state (guarded by `data_guard_`). - - // def_kind_ is not guard by DataGuard() since it is accessed in Init without - // locks. - uint8_t def_kind_ : 2; - // Has this flag's value been modified? - bool modified_ : 1 ABSL_GUARDED_BY(DataGuard()); - // Has this flag been specified on command line. - bool on_command_line_ : 1 ABSL_GUARDED_BY(DataGuard()); + // Per-flag mutex guard, placement-initialized in Init(). + alignas(sizeof(void*)) mutable char data_guard_[sizeof(absl::Mutex)]{}; // Unique tag for absl::call_once call to initialize this flag. absl::once_flag init_control_; - // Sequence lock / mutation counter. - flags_internal::SequenceLock seq_lock_; + // Deduplicated filename index assigned on registration (0 = unset). + uint16_t file_id_; - // Optional flag's callback and absl::Mutex to guard the invocations. - FlagCallback* callback_ ABSL_GUARDED_BY(DataGuard()); - // Either a pointer to the function generating the default value based on the - // value specified in ABSL_FLAG or pointer to the dynamically set default - // value via SetCommandLineOptionWithMode. def_kind_ is used to distinguish - // these two cases. - FlagDefaultSrc default_value_; + // Atomic state bitfields (def_kind, modified, on_command_line, has_callback, + // and saturating 3-bit mod_count where 7 indicates overflow in side table). + // All writes occur under DataGuard() with release semantics; SaveState() and + // RestoreState() perform lock-free acquire loads to fast-path unmodified + // flags without triggering lazy initialization. + std::atomic<MuState> mu_state_; - // This is reserved space for an absl::Mutex to guard flag data. It will be - // initialized in FlagImpl::Init via placement new. - // We can't use "absl::Mutex data_guard_", since this class is not literal. - // We do not want to use "absl::Mutex* data_guard_", since this would require - // heap allocation during initialization, which is both slows program startup - // and can fail. Using reserved space + placement new allows us to avoid both - // problems. - alignas(absl::Mutex) mutable unsigned char data_guard_[sizeof(absl::Mutex)]; + // Spare trailing byte in FlagImpl (used by Step 6 for inline bool storage). + uint8_t reserved_; }; #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop @@ -791,25 +884,41 @@ // The Flag object parameterized by the flag's value type. This class implements // flag reflection handle interface. +template <typename T, + FlagValueStorageKind Kind = flags_internal::StorageKind<T>()> +struct FlagStorage { + constexpr explicit FlagStorage(const FlagMetaHeader* meta) + : impl_(meta), value_() {} + constexpr FlagStorage(const FlagMeta* meta, FlagHelpKind help_kind, + FlagDefaultKind default_kind) + : impl_(meta, help_kind, Kind, default_kind), value_() {} + + // The implementation depends on value_ field to be placed exactly after the + // impl_ field, so that impl_ can figure out the offset to the value and + // access it. + FlagImpl impl_; + FlagValue<T, Kind> value_; +}; + template <typename T> -class Flag { +class Flag : private FlagStorage<T> { public: - constexpr Flag(const char* name, const char* type_name, const char* filename, - FlagHelpArg help, const FlagDefaultArg default_arg) - : impl_(name, type_name, filename, &FlagOps<T>, help, - flags_internal::StorageKind<T>(), default_arg), - value_() {} + constexpr explicit Flag(const FlagMetaHeader* meta) : FlagStorage<T>(meta) {} + + constexpr Flag(const FlagMeta* meta, FlagHelpKind help_kind, + FlagDefaultKind default_kind) + : FlagStorage<T>(meta, help_kind, default_kind) {} // CommandLineFlag interface - absl::string_view Name() const { return impl_.Name(); } - std::string Filename() const { return impl_.Filename(); } - std::string Help() const { return impl_.Help(); } + absl::string_view Name() const { return this->impl_.Name(); } + std::string Filename() const { return this->impl_.Filename(); } + std::string Help() const { return this->impl_.Help(); } // Do not use. To be removed. bool IsSpecifiedOnCommandLine() const { - return impl_.IsSpecifiedOnCommandLine(); + return this->impl_.IsSpecifiedOnCommandLine(); } - std::string DefaultValue() const { return impl_.DefaultValue(); } - std::string CurrentValue() const { return impl_.CurrentValue(); } + std::string DefaultValue() const { return this->impl_.DefaultValue(); } + std::string CurrentValue() const { return this->impl_.CurrentValue(); } private: template <typename, bool> @@ -826,28 +935,21 @@ U u; #if !defined(NDEBUG) - impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + this->impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); #endif - if (ABSL_PREDICT_FALSE(!value_.Get(impl_.seq_lock_, u.value))) { - impl_.Read(&u.value); + if (ABSL_PREDICT_FALSE(!this->value_.Get(u.value))) { + this->impl_.Read(&u.value); } return std::move(u.value); } void Set(const T& v) { - impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); - impl_.Write(&v); + this->impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + this->impl_.Write(&v); } // Access to the reflection. - const CommandLineFlag& Reflect() const { return impl_; } - - // Flag's data - // The implementation depends on value_ field to be placed exactly after the - // impl_ field, so that impl_ can figure out the offset to the value and - // access it. - FlagImpl impl_; - FlagValue<T> value_; + const CommandLineFlag& Reflect() const { return this->impl_; } }; /////////////////////////////////////////////////////////////////////////////// @@ -895,6 +997,12 @@ *static_cast<T*>(v2) = *static_cast<const T*>(v1); return nullptr; case FlagOp::kCopyConstruct: + if constexpr (std::is_default_constructible_v<T>) { + if (v1 == nullptr) { + new (v2) T(); + return nullptr; + } + } new (v2) T(*static_cast<const T*>(v1)); return nullptr; case FlagOp::kSizeof: @@ -925,6 +1033,9 @@ size_t offset = (sizeof(FlagImpl) + round_to - 1) / round_to * round_to; return reinterpret_cast<void*>(offset); } + case FlagOp::kTypeName: + *static_cast<absl::string_view*>(v2) = GenTypeName<T>(); + return nullptr; } return nullptr; } @@ -939,8 +1050,12 @@ public: constexpr explicit FlagRegistrar(Flag<T>& flag, const char* filename) : flag_(flag) { - if (do_register) + if (filename != nullptr) { + flag_.impl_.SetFilename(filename); + } + if (do_register) { flags_internal::RegisterCommandLineFlag(flag_.impl_, filename); + } } FlagRegistrar OnUpdate(FlagCallbackFunc cb) && {
diff --git a/absl/flags/internal/sequence_lock.h b/absl/flags/internal/sequence_lock.h index 1729d7f..f3c95ec 100644 --- a/absl/flags/internal/sequence_lock.h +++ b/absl/flags/internal/sequence_lock.h
@@ -66,11 +66,6 @@ lock_.store(0, std::memory_order_release); } - // Returns true if the flag is either uninitialized or initialized with zero - // modifications. - bool IsUnmodified() const { - return lock_.load(std::memory_order_acquire) <= 0; - } // Copy "size" bytes of data from "src" to "dst", protected as a read-side // critical section of the sequence lock. @@ -124,25 +119,6 @@ lock_.store(orig_seq + 2, std::memory_order_release); } - // Return the number of times that Write() has been called. - // - // REQUIRES: This must be externally synchronized against concurrent calls to - // `Write()` or `IncrementModificationCount()`. - // REQUIRES: `MarkInitialized()` must have been previously called. - int64_t ModificationCount() const { - int64_t val = lock_.load(std::memory_order_relaxed); - assert(val != kUninitialized && (val & 1) == 0); - return val / 2; - } - - // REQUIRES: This must be externally synchronized against concurrent calls to - // `Write()` or `ModificationCount()`. - // REQUIRES: `MarkInitialized()` must have been previously called. - void IncrementModificationCount() { - int64_t val = lock_.load(std::memory_order_relaxed); - assert(val != kUninitialized); - lock_.store(val + 2, std::memory_order_relaxed); - } private: // Perform the equivalent of "memcpy(dst, src, size)", but using relaxed