Emit pointer-free flat .rodata descriptors for scalars, 16-byte types, and empty defaults while keeping help strings cold. Pointer-bearing FlagMeta descriptors must live in .data.rel.ro and carry dynamic relocations for the flag name, help string, and default generator function. For built-in scalars, 16-byte built-ins (int128, uint128, and Time), and built-in flags whose default value is empty, all descriptor contents are compile-time constants. Emit 1-byte-aligned pointer-free flat descriptors (ZeroFlatFlagMeta, OneWordFlatFlagMeta, and TwoWordFlatFlagMeta) containing only FlagMetaHeader, inline default bytes when non-empty, and the NUL-terminated flag name. Because these structs contain no pointers, the compiler places them in shareable .rodata with no .data.rel.ro footprint and no dynamic relocations. Help strings remain in the cold flags_help_cold section and are registered into a compact offset table in flag.cc via FlagRegistrar::SetColdHelp() without dereferencing the cold page at startup. PiperOrigin-RevId: 1006216627
diff --git a/absl/flags/BUILD.bazel b/absl/flags/BUILD.bazel index 59771c7..b1ed5f9 100644 --- a/absl/flags/BUILD.bazel +++ b/absl/flags/BUILD.bazel
@@ -239,12 +239,16 @@ "//absl/base:core_headers", "//absl/base:dynamic_annotations", "//absl/base:fast_type_id", + "//absl/base:log_severity", "//absl/base:no_destructor", + "//absl/container:flat_hash_map", "//absl/memory", "//absl/meta:type_traits", + "//absl/numeric:int128", "//absl/strings", "//absl/strings:string_view", "//absl/synchronization", + "//absl/time", "//absl/utility", ], )
diff --git a/absl/flags/CMakeLists.txt b/absl/flags/CMakeLists.txt index 162b654..e364ad3 100644 --- a/absl/flags/CMakeLists.txt +++ b/absl/flags/CMakeLists.txt
@@ -201,9 +201,13 @@ absl::flags_commandlineflag_internal absl::flags_config absl::flags_marshalling + absl::flat_hash_map + absl::int128 + absl::log_severity absl::meta absl::no_destructor absl::synchronization + absl::time absl::utility PUBLIC )
diff --git a/absl/flags/flag.h b/absl/flags/flag.h index 2e9a9e6..3bc4a4d 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h
@@ -197,11 +197,11 @@ // ABSL_FLAG_IMPL macro definition conditional on ABSL_FLAGS_STRIP_NAMES #define ABSL_FLAG_IMPL_FLAG_PTR(flag) flag -#define ABSL_FLAG_IMPL_HELP_ARG(name) \ - absl::flags_internal::HelpArg<AbslFlagHelpGenFor##name>( \ +#define ABSL_FLAG_IMPL_HELP_ARG(name) \ + ::absl::flags_internal::HelpArg<AbslFlagHelpGenFor##name>( \ FLAGS_help_storage_##name) #define ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name) \ - absl::flags_internal::DefaultArg<Type, AbslFlagDefaultGenFor##name>(0) + ::absl::flags_internal::DefaultArg<Type, AbslFlagDefaultGenFor##name>(0) #if ABSL_FLAGS_STRIP_NAMES #define ABSL_FLAG_IMPL_FLAGNAME(txt) "" @@ -210,6 +210,9 @@ #define ABSL_FLAG_IMPL_REGISTRAR(T, flag) \ absl::flags_internal::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(flag), \ nullptr) +#define ABSL_FLAG_IMPL_REGISTRAR_WITH_HELP(T, flag, cold_help) \ + absl::flags_internal::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(flag), \ + nullptr, nullptr) #else #define ABSL_FLAG_IMPL_FLAGNAME(txt) txt #define ABSL_FLAG_IMPL_TYPENAME(txt) txt @@ -217,6 +220,9 @@ #define ABSL_FLAG_IMPL_REGISTRAR(T, flag) \ absl::flags_internal::FlagRegistrar<T, true>(ABSL_FLAG_IMPL_FLAG_PTR(flag), \ __FILE__) +#define ABSL_FLAG_IMPL_REGISTRAR_WITH_HELP(T, flag, cold_help) \ + absl::flags_internal::FlagRegistrar<T, true>(ABSL_FLAG_IMPL_FLAG_PTR(flag), \ + __FILE__, cold_help) #endif // ABSL_FLAG_IMPL macro definition conditional on ABSL_FLAGS_STRIP_HELP @@ -258,6 +264,22 @@ static void Gen(void* absl_flag_default_loc) { \ new (absl_flag_default_loc) Type(AbslFlagDefaultGenFor##name{}.value); \ } \ + template <typename AbslDefT = Type> \ + static constexpr bool IsEmpty( \ + bool absl_is_empty = \ + absl::flags_internal::IsEmptyDefaultArg<AbslDefT>( \ + default_value)) { \ + return absl_is_empty; \ + } \ + }; + +#define ABSL_FLAG_IMPL_DECLARE_META_WRAPPER(name) \ + template <typename AbslFlagType> \ + struct AbslFlagMetaFor##name { \ + static constexpr auto kMeta = ::absl::flags_internal::SelectFlagMeta< \ + AbslFlagType, AbslFlagDefaultGenFor##name, AbslFlagHelpGenFor##name, \ + sizeof(ABSL_FLAG_IMPL_FLAGNAME(#name))>( \ + ABSL_FLAG_IMPL_FLAGNAME(#name), FLAGS_help_storage_##name, 0); \ }; // ABSL_FLAG_IMPL @@ -265,18 +287,19 @@ // Note: Name of registrar object is not arbitrary. It is used to "grab" // global name for FLAGS_no<flag_name> symbol, thus preventing the possibility // of defining two flags with names foo and nofoo. -#define ABSL_FLAG_IMPL(Type, name, default_value, help) \ - extern ::absl::Flag<Type> FLAGS_##name; \ - namespace absl /* block flags in namespaces */ {} \ - ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ - ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, help) \ - ABSL_CONST_INIT absl::Flag<Type> FLAGS_##name{ \ - ABSL_FLAG_IMPL_FLAGNAME(#name), ABSL_FLAG_IMPL_TYPENAME(#Type), \ - ABSL_FLAG_IMPL_FILENAME(), ABSL_FLAG_IMPL_HELP_ARG(name), \ - ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name)}; \ - extern absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name; \ - absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name = \ - ABSL_FLAG_IMPL_REGISTRAR(Type, FLAGS_##name) +#define ABSL_FLAG_IMPL(Type, name, default_value, help) \ + extern ::absl::Flag<Type> FLAGS_##name; \ + namespace absl /* block flags in namespaces */ {} \ + ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ + ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, help) \ + ABSL_FLAG_IMPL_DECLARE_META_WRAPPER(name) \ + ABSL_CONST_INIT absl::Flag<Type> FLAGS_##name{ \ + &AbslFlagMetaFor##name<Type>::kMeta.hdr}; \ + extern absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name; \ + absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name = \ + ABSL_FLAG_IMPL_REGISTRAR_WITH_HELP( \ + Type, FLAGS_##name, \ + ::absl::flags_internal::ColdHelpPtr(FLAGS_help_storage_##name)) // ABSL_RETIRED_FLAG //
diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc index 5cacf64..3cc031c 100644 --- a/absl/flags/flag_test.cc +++ b/absl/flags/flag_test.cc
@@ -110,7 +110,7 @@ EXPECT_EQ(flags::StorageKind<int>(), flags::FlagValueStorageKind::kValueAndInitBit); EXPECT_EQ(flags::StorageKind<bool>(), - flags::FlagValueStorageKind::kValueAndInitBit); + flags::FlagValueStorageKind::kInlineBool); EXPECT_EQ(flags::StorageKind<double>(), flags::FlagValueStorageKind::kOneWordAtomic); EXPECT_EQ(flags::StorageKind<int64_t>(), @@ -137,6 +137,20 @@ flags::FlagValueStorageKind::kSequenceLocked); EXPECT_EQ(flags::StorageKind<absl::uint128>(), flags::FlagValueStorageKind::kSequenceLocked); + + // These sizes are not hard limits, but we want to keep per-flag size as + // small as possible because production binaries may define thousands of + // flags. + if (sizeof(void*) == 8) { + EXPECT_EQ(sizeof(flags::FlagImpl), 32); + EXPECT_EQ(sizeof(flags::FlagMetaHeader), 3); + EXPECT_EQ(sizeof(flags::BuiltinPtrFlagMeta), 32); + EXPECT_EQ(sizeof(flags::FlagMeta), 40); + EXPECT_EQ(sizeof(absl::Flag<bool>), 32); + EXPECT_EQ(sizeof(absl::Flag<int>), 40); + EXPECT_EQ(sizeof(absl::Flag<int64_t>), 40); + EXPECT_EQ(sizeof(absl::Flag<double>), 40); + } } // -------------------------------------------------------------------- @@ -148,17 +162,19 @@ using int128 = absl::int128; using uint128 = absl::uint128; -#define DEFINE_CONSTRUCTED_FLAG(T, dflt, dflt_kind) \ - constexpr flags::FlagDefaultArg f1default##T{ \ - flags::FlagDefaultSrc{dflt}, flags::FlagDefaultKind::dflt_kind}; \ - constexpr absl::Flag<T> f1##T{"f1", #T, "file", help_arg, f1default##T}; \ - ABSL_CONST_INIT absl::Flag<T> f2##T { \ - "f2", #T, "file", \ - {flags::FlagHelpMsg(&TestHelpMsg), flags::FlagHelpKind::kGenFunc}, \ - flags::FlagDefaultArg { \ - flags::FlagDefaultSrc(&TestMakeDflt<T>), \ - flags::FlagDefaultKind::kGenFunc \ - } \ +#define DEFINE_CONSTRUCTED_FLAG(T, dflt, dflt_kind) \ + constexpr flags::FlagDefaultArg f1default##T{ \ + flags::FlagDefaultSrc{dflt}, flags::FlagDefaultKind::dflt_kind}; \ + constexpr flags::FlagMeta f1meta##T = \ + flags::MakeFlagMeta<T>("f1", help_arg, f1default##T); \ + constexpr absl::Flag<T> f1##T{&f1meta##T, help_arg.kind, f1default##T.kind}; \ + constexpr flags::FlagMeta f2meta##T = flags::MakeFlagMeta<T>( \ + "f2", {flags::FlagHelpMsg(&TestHelpMsg), flags::FlagHelpKind::kGenFunc}, \ + flags::FlagDefaultArg{flags::FlagDefaultSrc(&TestMakeDflt<T>), \ + flags::FlagDefaultKind::kGenFunc}); \ + ABSL_CONST_INIT absl::Flag<T> f2##T { \ + &f2meta##T, flags::FlagHelpKind::kGenFunc, \ + flags::FlagDefaultKind::kGenFunc \ } DEFINE_CONSTRUCTED_FLAG(bool, true, kOneWord); @@ -179,9 +195,9 @@ bool TestConstructionFor(const absl::Flag<T>& f1, absl::Flag<T>& f2) { EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Name(), "f1"); EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Help(), "literal help"); - EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Filename(), "file"); + EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Filename(), ""); - flags::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(f2), nullptr) + flags::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(f2), "file") .OnUpdate(TestCallback); EXPECT_EQ(absl::GetFlagReflectionHandle(f2).Name(), "f2"); @@ -1397,3 +1413,48 @@ #endif } // namespace + +ABSL_FLAG(int, concurrent_saver_test_flag, 0, + "test flag for concurrent FlagSaver"); + +namespace { + +TEST_F(FlagTest, ConcurrentFlagSaverAndSetFlag) { + std::atomic<bool> stop{false}; + std::thread writer([&stop]() { + int i = 1; + while (!stop.load(std::memory_order_relaxed)) { + absl::SetFlag(&FLAGS_concurrent_saver_test_flag, i++); + } + }); + for (int iter = 0; iter < 50; ++iter) { + absl::FlagSaver saver; + (void)absl::GetFlag(FLAGS_concurrent_saver_test_flag); + } + stop.store(true, std::memory_order_relaxed); + writer.join(); + EXPECT_GE(absl::GetFlag(FLAGS_concurrent_saver_test_flag), 0); +} + +TEST_F(FlagTest, FilenameInterning) { + constexpr const char* kFileA = "dir/intern_test_a.cc"; + constexpr const char* kFileB = "dir/intern_test_b.cc"; + + absl::Flag<int32_t> f_a1{&f1metaint32_t, help_arg.kind, + f1defaultint32_t.kind}; + absl::Flag<int32_t> f_a2{&f1metaint32_t, help_arg.kind, + f1defaultint32_t.kind}; + absl::Flag<int32_t> f_b{&f1metaint32_t, help_arg.kind, f1defaultint32_t.kind}; + + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_a1), kFileA); + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_a2), kFileA); + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_b), kFileB); + // Duplicate registration preserves the first filename for ODR diagnostics. + flags::FlagRegistrar<int32_t, false>(ABSL_FLAG_IMPL_FLAG_PTR(f_a1), kFileB); + + EXPECT_EQ(f_a1.Filename(), "dir/intern_test_a.cc"); + EXPECT_EQ(f_a2.Filename(), "dir/intern_test_a.cc"); + EXPECT_EQ(f_b.Filename(), "dir/intern_test_b.cc"); +} + +} // namespace
diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index 662b342..3d208ae 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc
@@ -23,7 +23,9 @@ #include <array> #include <atomic> #include <cstring> +#include <limits> #include <memory> +#include <optional> #include <string> #include <typeinfo> #include <utility> @@ -36,16 +38,20 @@ #include "absl/base/const_init.h" #include "absl/base/dynamic_annotations.h" #include "absl/base/fast_type_id.h" +#include "absl/base/log_severity.h" #include "absl/base/no_destructor.h" #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" #include "absl/flags/config.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/flags/usage_config.h" #include "absl/memory/memory.h" +#include "absl/numeric/int128.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -104,6 +110,163 @@ s_freelist->push_back(p); } +template <typename T> +struct FlagSideTable { + absl::Mutex mutex{absl::kConstInit}; + absl::flat_hash_map<const FlagImpl*, T>* table ABSL_GUARDED_BY(mutex) = + nullptr; + + T& operator[](const FlagImpl* flag) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { + if (table == nullptr) { + table = new absl::flat_hash_map<const FlagImpl*, T>; + } + return (*table)[flag]; + } + + T Get(const FlagImpl* flag) { + absl::MutexLock l(mutex); + if (table == nullptr) return T{}; + auto it = table->find(flag); + return it != table->end() ? it->second : T{}; + } +}; + +ABSL_CONST_INIT FlagSideTable<FlagCallback*> callbacks; +ABSL_CONST_INIT FlagSideTable<void*> dynamic_defaults; +ABSL_CONST_INIT FlagSideTable<int64_t> overflow_mod_counts; + +FlagCallback* GetOrCreateCallback(const FlagImpl* flag) { + absl::MutexLock l(callbacks.mutex); + FlagCallback*& cb = callbacks[flag]; + if (cb == nullptr) { + cb = new FlagCallback; + } + return cb; +} + +void* GetDynamicDefault(const FlagImpl* flag) { + return dynamic_defaults.Get(flag); +} + +void* SetDynamicDefault(const FlagImpl* flag, void* new_val) { + absl::MutexLock l(dynamic_defaults.mutex); + return std::exchange(dynamic_defaults[flag], new_val); +} + +int64_t GetOverflowModCount(const FlagImpl* flag) { + return overflow_mod_counts.Get(flag); +} + +void IncrementOverflowModCount(const FlagImpl* flag) { + absl::MutexLock l(overflow_mod_counts.mutex); + ++overflow_mod_counts[flag]; +} + +ABSL_INTERNAL_DISABLE_DEPRECATED_DECLARATION_WARNING +constexpr FlagOpFn kBuiltinFlagOps[] = { + &FlagOps<bool>, + &FlagOps<short>, // NOLINT + &FlagOps<unsigned short>, // NOLINT + &FlagOps<int>, // NOLINT + &FlagOps<unsigned int>, // NOLINT + &FlagOps<long>, // NOLINT + &FlagOps<unsigned long>, // NOLINT + &FlagOps<long long>, // NOLINT + &FlagOps<unsigned long long>, // NOLINT + &FlagOps<float>, + &FlagOps<double>, + &FlagOps<std::string>, + &FlagOps<std::vector<std::string>>, + &FlagOps<absl::int128>, + &FlagOps<absl::uint128>, + &FlagOps<absl::LogSeverity>, + &FlagOps<std::optional<bool>>, + &FlagOps<std::optional<int>>, + &FlagOps<std::optional<long>>, // NOLINT + &FlagOps<std::optional<long long>>, // NOLINT + &FlagOps<std::optional<unsigned long>>, // NOLINT + &FlagOps<std::optional<double>>, + &FlagOps<std::optional<std::string>>, + &FlagOps<absl::Duration>, + &FlagOps<absl::Time>, + &FlagOps<std::optional<absl::Duration>>, +}; +ABSL_INTERNAL_RESTORE_DEPRECATED_DECLARATION_WARNING + +struct FilenameTable { + absl::Mutex mutex; + std::vector<const char*> files ABSL_GUARDED_BY(mutex); +}; + +FilenameTable& GlobalFilenames() { + static absl::NoDestructor<FilenameTable> filenames; + return *filenames; +} + +const char* GetFilenameById(uint16_t id) { + if (id == 0) return ""; + auto& table = GlobalFilenames(); + absl::MutexLock l(table.mutex); + return table.files[id - 1]; +} + +struct ColdHelpBase { + uintptr_t flag_base = 0; + uintptr_t help_base = 0; + std::vector<std::pair<uint32_t, uint32_t>> entries; + std::vector<std::pair<const FlagImpl*, const char*>> overflow; +}; + +struct ColdHelpRegistry { + absl::Mutex mutex{absl::kConstInit}; + ColdHelpBase* data ABSL_GUARDED_BY(mutex) = nullptr; +}; + +ABSL_CONST_INIT ColdHelpRegistry cold_help_registry; + +void RegisterColdHelp(const FlagImpl* flag, const char* cold_help) { + absl::MutexLock l(cold_help_registry.mutex); + if (cold_help_registry.data == nullptr) { + cold_help_registry.data = new ColdHelpBase; + } + auto& t = *cold_help_registry.data; + const uintptr_t fa = reinterpret_cast<uintptr_t>(flag); + const uintptr_t ha = reinterpret_cast<uintptr_t>(cold_help); + if (t.entries.empty() && t.overflow.empty()) { + t.flag_base = fa; + t.help_base = ha; + t.entries.reserve(1024); + } + const uintptr_t fd = fa - t.flag_base; + const uintptr_t hd = ha - t.help_base; + if (fd <= 0xFFFFFFFFu && hd <= 0xFFFFFFFFu) { + t.entries.emplace_back(static_cast<uint32_t>(fd), + static_cast<uint32_t>(hd)); + } else { + t.overflow.emplace_back(flag, cold_help); + } +} + +const char* GetColdHelp(const FlagImpl* flag) { + absl::MutexLock l(cold_help_registry.mutex); + if (cold_help_registry.data == nullptr) return ""; + const auto& t = *cold_help_registry.data; + const uintptr_t fa = reinterpret_cast<uintptr_t>(flag); + const uintptr_t fd = fa - t.flag_base; + if (fd <= 0xFFFFFFFFu) { + const uint32_t target = static_cast<uint32_t>(fd); + for (const auto& entry : t.entries) { + if (entry.first == target) { + return reinterpret_cast<const char*>(t.help_base + entry.second); + } + } + } + for (const auto& entry : t.overflow) { + if (entry.first == flag) return entry.second; + } + return ""; +} + } // namespace /////////////////////////////////////////////////////////////////////////////// @@ -129,9 +292,9 @@ on_command_line_(on_command_line), counter_(counter) {} - FlagState(FlagImpl& flag_impl, FlagDefaultSrc default_value) + explicit FlagState(FlagImpl& flag_impl) : flag_impl_(flag_impl), - value_(default_value), + value_(static_cast<int64_t>(0)), modified_(false), on_command_line_(false), counter_(-1) {} @@ -141,7 +304,7 @@ if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kHeapAllocated && flag_impl_.ValueStorageKind() != FlagValueStorageKind::kSequenceLocked) return; - flags_internal::Delete(flag_impl_.op_, value_.heap_allocated); + flags_internal::Delete(flag_impl_.op(), value_.heap_allocated); } private: @@ -161,11 +324,9 @@ union SavedValue { explicit SavedValue(void* v) : heap_allocated(v) {} explicit SavedValue(int64_t v) : one_word(v) {} - explicit SavedValue(FlagDefaultSrc v) : default_value(v) {} void* heap_allocated; int64_t one_word; - FlagDefaultSrc default_value; } value_; bool modified_; bool on_command_line_; @@ -210,25 +371,92 @@ return (reinterpret_cast<mask_t>(ptr_) & mask) != 0; } +void FlagImpl::SetFilename(const char* filename) { + // Keep the first registration's filename so FlagRegistry::RegisterFlag can + // report both files when dynamic symbol interposition merges duplicate flag + // definitions across shared libraries. + if (file_id_ != 0 || filename == nullptr) return; + auto& table = GlobalFilenames(); + absl::MutexLock l(table.mutex); + // All flags in a translation unit register consecutively from one + // .init_array initializer and share one merged .rodata.str1.1 __FILE__ + // string literal, so pointer equality against the last entry deduplicates + // per-TU filenames in O(1) without strcmp or a hash table. + if (!table.files.empty() && table.files.back() == filename) { + file_id_ = static_cast<uint16_t>(table.files.size()); + return; + } + if (ABSL_PREDICT_FALSE(table.files.size() >= + std::numeric_limits<uint16_t>::max())) { + ABSL_INTERNAL_LOG(FATAL, "Too many flag filenames"); + } + table.files.push_back(filename); + file_id_ = static_cast<uint16_t>(table.files.size()); +} + +void FlagImpl::SetColdHelp(const char* cold_help) { + if (cold_help != nullptr && MetaKind() != FlagMetaKind::kPtrMeta) { + RegisterColdHelp(this, cold_help); + } +} + +FlagOpFn FlagImpl::op() const { + uint8_t id = meta_->builtin_op_id; + if (ABSL_PREDICT_TRUE(id != kNoBuiltinOp)) { + return kBuiltinFlagOps[id]; + } + return reinterpret_cast<const FlagMeta*>(meta_)->op; +} + +const void* FlagImpl::DefaultValueStoragePtr() const { + if (MetaKind() == FlagMetaKind::kPtrMeta) { + return &PtrMeta()->default_value; + } + return reinterpret_cast<const char*>(meta_) + sizeof(FlagMetaHeader); +} + +const char* FlagImpl::DefaultCStringPtr() const { + return Name().data() + meta_->name_len + 1; +} + void FlagImpl::Init() { new (&data_guard_) absl::Mutex; - auto def_kind = static_cast<FlagDefaultKind>(def_kind_); + FlagDefaultKind def_kind = + mu_state_.load(std::memory_order_relaxed).DefaultKind(); switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: { + bool val = false; + if (def_kind == FlagDefaultKind::kGenFunc) { + (*PtrMeta()->default_value.gen_func)(&val); + } else if (def_kind == FlagDefaultKind::kOneWord) { + if (MetaKind() == FlagMetaKind::kPtrMeta) { + std::memcpy(&val, &PtrMeta()->default_value, sizeof(bool)); + } else { + val = true; + } + } + ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(&val, sizeof(bool)); + inline_bool_value_.store( + static_cast<uint8_t>((static_cast<uint8_t>(val) & 1u) | 2u), + std::memory_order_release); + break; + } case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { alignas(int64_t) std::array<char, sizeof(int64_t)> buf{}; if (def_kind == FlagDefaultKind::kGenFunc) { - (*default_value_.gen_func)(buf.data()); + (*PtrMeta()->default_value.gen_func)(buf.data()); + } else if (def_kind == FlagDefaultKind::kOneWord) { + std::memcpy(buf.data(), DefaultValueStoragePtr(), Sizeof(op())); } else { - assert(def_kind != FlagDefaultKind::kDynamicValue); - std::memcpy(buf.data(), &default_value_, Sizeof(op_)); + assert(def_kind == FlagDefaultKind::kDefaultConstructed); } if (ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit) { // We presume here the memory layout of FlagValueAndInitBit struct. uint8_t initialized = 1; - std::memcpy(buf.data() + Sizeof(op_), &initialized, + std::memcpy(buf.data() + Sizeof(op()), &initialized, sizeof(initialized)); } // Type can contain valid uninitialized bits, e.g. padding. @@ -238,38 +466,48 @@ break; } case FlagValueStorageKind::kSequenceLocked: { - // For this storage kind the default_value_ always points to gen_func - // during initialization. - assert(def_kind == FlagDefaultKind::kGenFunc); - (*default_value_.gen_func)(AtomicBufferValue()); + if (def_kind == FlagDefaultKind::kGenFunc) { + (*PtrMeta()->default_value.gen_func)(AtomicBufferValue()); + } else if (def_kind == FlagDefaultKind::kOneWord) { + std::memcpy(static_cast<void*>(AtomicBufferValue()), + DefaultValueStoragePtr(), Sizeof(op())); + } else { + assert(def_kind == FlagDefaultKind::kDefaultConstructed); + std::memset(static_cast<void*>(AtomicBufferValue()), 0, Sizeof(op())); + } + SeqLock().MarkInitialized(); break; } - case FlagValueStorageKind::kHeapAllocated: - // For this storage kind the default_value_ always points to gen_func - // during initialization. - assert(def_kind == FlagDefaultKind::kGenFunc); + case FlagValueStorageKind::kHeapAllocated: { // Flag value initially points to the internal buffer. MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire); - (*default_value_.gen_func)(ptr_value.Ptr()); + if (def_kind == FlagDefaultKind::kGenFunc) { + (*PtrMeta()->default_value.gen_func)(ptr_value.Ptr()); + } else if (def_kind == FlagDefaultKind::kOneWord) { + ::new (ptr_value.Ptr()) std::string(DefaultCStringPtr()); + } else { + assert(def_kind == FlagDefaultKind::kDefaultConstructed); + flags_internal::CopyConstruct(op(), nullptr, ptr_value.Ptr()); + } // Default value is a candidate for an unprotected read. PtrStorage().store(MaskedPointer(ptr_value.Ptr(), true), std::memory_order_release); break; + } } - seq_lock_.MarkInitialized(); } -absl::Mutex& FlagImpl::DataGuard() const { +absl::Mutex* FlagImpl::DataGuard() const { absl::call_once(const_cast<FlagImpl*>(this)->init_control_, &FlagImpl::Init, const_cast<FlagImpl*>(this)); // data_guard_ is initialized inside Init. - return *reinterpret_cast<absl::Mutex*>(&data_guard_); + return reinterpret_cast<absl::Mutex*>(data_guard_); } void FlagImpl::AssertValidType(FlagFastTypeId rhs_type_id, const std::type_info* (*gen_rtti)()) const { - FlagFastTypeId lhs_type_id = flags_internal::FastTypeId(op_); + FlagFastTypeId lhs_type_id = flags_internal::FastTypeId(op()); // `rhs_type_id` is the fast type id corresponding to the declaration // visible at the call site. `lhs_type_id` is the fast type id @@ -278,7 +516,7 @@ if (ABSL_PREDICT_TRUE(lhs_type_id == rhs_type_id)) return; const std::type_info* lhs_runtime_type_id = - flags_internal::RuntimeTypeId(op_); + flags_internal::RuntimeTypeId(op()); const std::type_info* rhs_runtime_type_id = (*gen_rtti)(); if (lhs_runtime_type_id == rhs_runtime_type_id) return; @@ -294,34 +532,65 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const { void* res = nullptr; - switch (DefaultKind()) { + switch (mu_state_.load(std::memory_order_relaxed).DefaultKind()) { case FlagDefaultKind::kDynamicValue: - res = flags_internal::Clone(op_, default_value_.dynamic_value); + res = flags_internal::Clone(op(), GetDynamicDefault(this)); break; case FlagDefaultKind::kGenFunc: - res = flags_internal::Alloc(op_); - (*default_value_.gen_func)(res); + res = flags_internal::Alloc(op()); + (*PtrMeta()->default_value.gen_func)(res); break; - default: - res = flags_internal::Clone(op_, &default_value_); + case FlagDefaultKind::kDefaultConstructed: + res = flags_internal::Clone(op(), nullptr); + break; + case FlagDefaultKind::kOneWord: + if (ValueStorageKind() == FlagValueStorageKind::kInlineBool && + MetaKind() != FlagMetaKind::kPtrMeta) { + bool b = true; + res = flags_internal::Clone(op(), &b); + } else if (ValueStorageKind() == FlagValueStorageKind::kHeapAllocated && + MetaKind() != FlagMetaKind::kPtrMeta) { + res = flags_internal::Alloc(op()); + ::new (res) std::string(DefaultCStringPtr()); + } else { + res = flags_internal::Alloc(op()); + std::memcpy(res, DefaultValueStoragePtr(), Sizeof(op())); + } break; } - return {res, DynValueDeleter{op_}}; + return {res, DynValueDeleter{op()}}; +} + +void FlagImpl::RecordMutation() { + MuState s = mu_state_.load(std::memory_order_relaxed); + if (s.mod_count < kMaxInlineModCount) { + ++s.mod_count; + } else { + IncrementOverflowModCount(this); + } + s.modified = true; + mu_state_.store(s, std::memory_order_release); } void FlagImpl::StoreValue(const void* src, ValueSource source) { switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: { + bool val = *static_cast<const bool*>(src); + inline_bool_value_.store( + static_cast<uint8_t>((static_cast<uint8_t>(val) & 1u) | 2u), + std::memory_order_release); + break; + } case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { // Load the current value to avoid setting 'init' bit manually. int64_t one_word_val = OneWordValue().load(std::memory_order_acquire); - std::memcpy(&one_word_val, src, Sizeof(op_)); + std::memcpy(&one_word_val, src, Sizeof(op())); OneWordValue().store(one_word_val, std::memory_order_release); - seq_lock_.IncrementModificationCount(); break; } case FlagValueStorageKind::kSequenceLocked: { - seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_)); + SeqLock().Write(AtomicBufferValue(), src, Sizeof(op())); break; } case FlagValueStorageKind::kHeapAllocated: @@ -335,76 +604,111 @@ // instead we move the memory to a freelist so it can still be safely // accessed, and allocate a new one for the new value. AddToFreelist(ptr_value.Ptr()); - ptr_value = MaskedPointer(Clone(op_, src), source == kCommandLine); + ptr_value = MaskedPointer(Clone(op(), src), source == kCommandLine); } else { // Current value either was set programmatically or was never read. // We can reuse the memory since all accesses to this value (if any) // were protected by mutex. That said, if a new value comes from command // line it now becomes a candidate for an unprotected read. - ptr_value.Set(op_, src, source == kCommandLine); + ptr_value.Set(op(), src, source == kCommandLine); } PtrStorage().store(ptr_value, std::memory_order_release); - seq_lock_.IncrementModificationCount(); break; } - modified_ = true; + RecordMutation(); InvokeCallback(); } -absl::string_view FlagImpl::Name() const { return name_; } +absl::string_view FlagImpl::Name() const { + const char* base = reinterpret_cast<const char*>(meta_); + switch (MetaKind()) { + case FlagMetaKind::kPtrMeta: + return PtrMeta()->name; + case FlagMetaKind::kZeroFlatMeta: + return absl::string_view(base + sizeof(FlagMetaHeader), meta_->name_len); + case FlagMetaKind::kOneWordFlatMeta: + if (ValueStorageKind() == FlagValueStorageKind::kHeapAllocated) { + return absl::string_view(base + sizeof(FlagMetaHeader), + meta_->name_len); + } + return absl::string_view(base + sizeof(FlagMetaHeader) + 8, + meta_->name_len); + case FlagMetaKind::kTwoWordFlatMeta: + return absl::string_view(base + sizeof(FlagMetaHeader) + 16, + meta_->name_len); + } + return ""; +} -absl::string_view FlagImpl::TypeName() const { return type_name_; } +absl::string_view FlagImpl::TypeName() const { + return flags_internal::TypeName(op()); +} std::string FlagImpl::Filename() const { - return flags_internal::GetUsageConfig().normalize_filename(filename_); + return flags_internal::GetUsageConfig().normalize_filename( + GetFilenameById(file_id_)); } std::string FlagImpl::Help() const { - return HelpSourceKind() == FlagHelpKind::kLiteral ? help_.literal - : help_.gen_func(); + if (MetaKind() == FlagMetaKind::kPtrMeta) { + return HelpSourceKind() == FlagHelpKind::kLiteral + ? PtrMeta()->help.literal + : PtrMeta()->help.gen_func(); + } + return std::string(GetColdHelp(this)); } FlagFastTypeId FlagImpl::TypeId() const { - return flags_internal::FastTypeId(op_); + return flags_internal::FastTypeId(op()); } int64_t FlagImpl::ModificationCount() const { - return seq_lock_.ModificationCount(); + const uint8_t mod_count = + mu_state_.load(std::memory_order_relaxed).mod_count; + if (ABSL_PREDICT_TRUE(mod_count < kMaxInlineModCount)) { + return mod_count; + } + return kMaxInlineModCount + GetOverflowModCount(this); } bool FlagImpl::IsSpecifiedOnCommandLine() const { - absl::MutexLock l(DataGuard()); - return on_command_line_; + absl::MutexLock l(*DataGuard()); + return mu_state_.load(std::memory_order_relaxed).on_command_line; } std::string FlagImpl::DefaultValue() const { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); auto obj = MakeInitValue(); - return flags_internal::Unparse(op_, obj.get()); + return flags_internal::Unparse(op(), obj.get()); } std::string FlagImpl::CurrentValue() const { - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: { + bool val = static_cast<bool>( + inline_bool_value_.load(std::memory_order_acquire) & 1u); + return flags_internal::Unparse(op(), &val); + } case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { - const auto one_word_val = + alignas(int64_t) const auto one_word_val = absl::bit_cast<std::array<char, sizeof(int64_t)>>( OneWordValue().load(std::memory_order_acquire)); - return flags_internal::Unparse(op_, one_word_val.data()); + return flags_internal::Unparse(op(), one_word_val.data()); } case FlagValueStorageKind::kSequenceLocked: { - std::unique_ptr<void, DynValueDeleter> cloned(flags_internal::Alloc(op_), - DynValueDeleter{op_}); + std::unique_ptr<void, DynValueDeleter> cloned(flags_internal::Alloc(op()), + DynValueDeleter{op()}); ReadSequenceLockedData(cloned.get()); - return flags_internal::Unparse(op_, cloned.get()); + return flags_internal::Unparse(op(), cloned.get()); } case FlagValueStorageKind::kHeapAllocated: { - absl::MutexLock l(guard); + absl::MutexLock l(*guard); return flags_internal::Unparse( - op_, PtrStorage().load(std::memory_order_acquire).Ptr()); + op(), PtrStorage().load(std::memory_order_acquire).Ptr()); } } @@ -412,22 +716,24 @@ } void FlagImpl::SetCallback(const FlagCallbackFunc mutation_callback) { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); - if (callback_ == nullptr) { - callback_ = new FlagCallback; - } - callback_->func = mutation_callback; + FlagCallback* cb = GetOrCreateCallback(this); + cb->func = mutation_callback; + MuState s = mu_state_.load(std::memory_order_relaxed); + s.has_callback = true; + mu_state_.store(s, std::memory_order_release); InvokeCallback(); } void FlagImpl::InvokeCallback() const { - if (!callback_) return; + if (!mu_state_.load(std::memory_order_relaxed).has_callback) return; + FlagCallback* cb_entry = GetOrCreateCallback(this); // Make a copy of the C-style function pointer that we are about to invoke // before we release the lock guarding it. - FlagCallbackFunc cb = callback_->func; + FlagCallbackFunc cb = cb_entry->func; // If the flag has a mutation callback this function invokes it. While the // callback is being invoked the primary flag's mutex is unlocked and it is @@ -440,45 +746,46 @@ // and it also can be different by the time the callback invocation is // completed. Requires that *primary_lock be held in exclusive mode; it may be // released and reacquired by the implementation. - MutexRelock relock(DataGuard()); - absl::MutexLock lock(callback_->guard); + MutexRelock relock(*DataGuard()); + absl::MutexLock lock(cb_entry->guard); cb(); } -// Returns true if an unmodified absl::Flag (i.e. `seq_lock_.IsUnmodified()`) -// can skip lazy initialization and state allocation in `SaveState()`, and -// instead reconstructs from the provided default value if modified before -// restoration. bool FlagImpl::CanSkipInitOnSave() const { - if (!seq_lock_.IsUnmodified()) return false; + const MuState state = mu_state_.load(std::memory_order_acquire); + if (state.mod_count != 0) return false; + const FlagDefaultKind def_kind = state.DefaultKind(); + if (def_kind == FlagDefaultKind::kDynamicValue) return false; switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: - // Only `kOneWord` defaults are inlinable compile-time constants; other - // kinds may have dynamic defaults or side effects. - return static_cast<FlagDefaultKind>(def_kind_) == - FlagDefaultKind::kOneWord; + return def_kind != FlagDefaultKind::kGenFunc; case FlagValueStorageKind::kSequenceLocked: - // Always uses a default-generator function when unmodified. return true; case FlagValueStorageKind::kHeapAllocated: - // Built-in types have side-effect-free defaults; user-defined types may - // have custom construction or validation side effects. - return !ShouldValidateFlagValue(flags_internal::FastTypeId(op_)); + return !ShouldValidateFlagValue(flags_internal::FastTypeId(op())); } return false; } std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { if (ABSL_PREDICT_TRUE(CanSkipInitOnSave())) { - return std::make_unique<FlagState>(*this, default_value_); + return std::make_unique<FlagState>(*this); } - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); - bool modified = modified_; - bool on_command_line = on_command_line_; + const MuState state = mu_state_.load(std::memory_order_relaxed); + bool modified = state.modified; + bool on_command_line = state.on_command_line; switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: { + int64_t val = static_cast<int64_t>( + inline_bool_value_.load(std::memory_order_acquire) & 1u); + return std::make_unique<FlagState>(*this, val, modified, on_command_line, + ModificationCount()); + } case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { return std::make_unique<FlagState>( @@ -486,10 +793,10 @@ on_command_line, ModificationCount()); } case FlagValueStorageKind::kSequenceLocked: { - void* cloned = flags_internal::Alloc(op_); + void* cloned = flags_internal::Alloc(op()); // Read is guaranteed to be successful because we hold the lock. bool success = - seq_lock_.TryRead(cloned, AtomicBufferValue(), Sizeof(op_)); + SeqLock().TryRead(cloned, AtomicBufferValue(), Sizeof(op())); assert(success); static_cast<void>(success); return std::make_unique<FlagState>(*this, cloned, modified, @@ -499,7 +806,7 @@ return std::make_unique<FlagState>( *this, flags_internal::Clone( - op_, PtrStorage().load(std::memory_order_acquire).Ptr()), + op(), PtrStorage().load(std::memory_order_acquire).Ptr()), modified, on_command_line, ModificationCount()); } } @@ -507,39 +814,40 @@ } bool FlagImpl::RestoreState(const FlagState& flag_state) { - if (ABSL_PREDICT_TRUE(flag_state.counter_ < 0 && seq_lock_.IsUnmodified())) { + if (ABSL_PREDICT_TRUE(flag_state.counter_ < 0 && IsUnmodifiedFast())) { return false; } - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); if (flag_state.counter_ == ModificationCount()) { return false; } - switch (ValueStorageKind()) { - case FlagValueStorageKind::kValueAndInitBit: - case FlagValueStorageKind::kOneWordAtomic: - StoreValue( - flag_state.counter_ < 0 - ? static_cast<const void*>(&flag_state.value_.default_value) - : static_cast<const void*>(&flag_state.value_.one_word), - kProgrammaticChange); - break; - case FlagValueStorageKind::kSequenceLocked: - case FlagValueStorageKind::kHeapAllocated: - if (flag_state.counter_ < 0) { - std::unique_ptr<void, DynValueDeleter> init_value( - flags_internal::Alloc(op_), DynValueDeleter{op_}); - (*flag_state.value_.default_value.gen_func)(init_value.get()); - StoreValue(init_value.get(), kProgrammaticChange); - } else { - StoreValue(flag_state.value_.heap_allocated, kProgrammaticChange); + if (flag_state.counter_ < 0) { + auto init_value = MakeInitValue(); + StoreValue(init_value.get(), kProgrammaticChange); + } else { + switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: { + bool val = static_cast<bool>(flag_state.value_.one_word != 0); + StoreValue(&val, kProgrammaticChange); + break; } - break; + case FlagValueStorageKind::kValueAndInitBit: + case FlagValueStorageKind::kOneWordAtomic: + StoreValue(&flag_state.value_.one_word, kProgrammaticChange); + break; + case FlagValueStorageKind::kSequenceLocked: + case FlagValueStorageKind::kHeapAllocated: + StoreValue(flag_state.value_.heap_allocated, kProgrammaticChange); + break; + } } - modified_ = flag_state.modified_; - on_command_line_ = flag_state.on_command_line_; + MuState s = mu_state_.load(std::memory_order_relaxed); + s.modified = flag_state.modified_; + s.on_command_line = flag_state.on_command_line_; + mu_state_.store(s, std::memory_order_release); return true; } @@ -548,7 +856,7 @@ StorageT* FlagImpl::OffsetValue() const { char* p = reinterpret_cast<char*>(const_cast<FlagImpl*>(this)); // The offset is deduced via Flag value type specific op_. - ptrdiff_t offset = flags_internal::ValueOffset(op_); + ptrdiff_t offset = flags_internal::ValueOffset(op()); return reinterpret_cast<StorageT*>(p + offset); } @@ -558,6 +866,14 @@ return OffsetValue<std::atomic<uint64_t>>(); } +flags_internal::SequenceLock& FlagImpl::SeqLock() const { + assert(ValueStorageKind() == FlagValueStorageKind::kSequenceLocked); + size_t num_words = flags_internal::AlignUp(Sizeof(op()), sizeof(uint64_t)) / + sizeof(uint64_t); + return *reinterpret_cast<flags_internal::SequenceLock*>(AtomicBufferValue() + + num_words); +} + std::atomic<int64_t>& FlagImpl::OneWordValue() const { assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic || ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); @@ -578,7 +894,7 @@ std::unique_ptr<void, DynValueDeleter> tentative_value = MakeInitValue(); std::string parse_err; - if (!flags_internal::Parse(op_, value, tentative_value.get(), &parse_err)) { + if (!flags_internal::Parse(op(), value, tentative_value.get(), &parse_err)) { absl::string_view err_sep = parse_err.empty() ? "" : "; "; err = absl::StrCat("Illegal value '", value, "' specified for flag '", Name(), "'", err_sep, parse_err); @@ -589,13 +905,18 @@ } void FlagImpl::Read(void* dst) const { - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized switch (ValueStorageKind()) { + case FlagValueStorageKind::kInlineBool: { + *static_cast<bool*>(dst) = static_cast<bool>( + inline_bool_value_.load(std::memory_order_acquire) & 1u); + break; + } case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { const int64_t one_word_val = OneWordValue().load(std::memory_order_acquire); - std::memcpy(dst, &one_word_val, Sizeof(op_)); + std::memcpy(dst, &one_word_val, Sizeof(op())); break; } case FlagValueStorageKind::kSequenceLocked: { @@ -603,10 +924,10 @@ break; } case FlagValueStorageKind::kHeapAllocated: { - absl::MutexLock l(guard); + absl::MutexLock l(*guard); MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire); - flags_internal::CopyConstruct(op_, ptr_value.Ptr(), dst); + flags_internal::CopyConstruct(op(), ptr_value.Ptr(), dst); // For unprotected read candidates, mark that the value as has been read. if (ptr_value.IsUnprotectedReadCandidate() && !ptr_value.HasBeenRead()) { @@ -621,43 +942,48 @@ int64_t FlagImpl::ReadOneWord() const { assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic || ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); - auto& guard = DataGuard(); // Make sure flag initialized + auto* guard = DataGuard(); // Make sure flag initialized (void)guard; return OneWordValue().load(std::memory_order_acquire); } bool FlagImpl::ReadOneBool() const { - assert(ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); - auto& guard = DataGuard(); // Make sure flag initialized + assert(ValueStorageKind() == FlagValueStorageKind::kInlineBool || + ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); + auto* guard = DataGuard(); // Make sure flag initialized (void)guard; + if (ValueStorageKind() == FlagValueStorageKind::kInlineBool) { + return static_cast<bool>( + inline_bool_value_.load(std::memory_order_acquire) & 1u); + } return absl::bit_cast<FlagValueAndInitBit<bool>>( OneWordValue().load(std::memory_order_acquire)) .value; } void FlagImpl::ReadSequenceLockedData(void* dst) const { - size_t size = Sizeof(op_); + size_t size = Sizeof(op()); // Attempt to read using the sequence lock. - if (ABSL_PREDICT_TRUE(seq_lock_.TryRead(dst, AtomicBufferValue(), size))) { + if (ABSL_PREDICT_TRUE(SeqLock().TryRead(dst, AtomicBufferValue(), size))) { return; } // We failed due to contention. Acquire the lock to prevent contention // and try again. - absl::ReaderMutexLock l(DataGuard()); - bool success = seq_lock_.TryRead(dst, AtomicBufferValue(), size); + absl::ReaderMutexLock l(*DataGuard()); + bool success = SeqLock().TryRead(dst, AtomicBufferValue(), size); assert(success); static_cast<void>(success); } void FlagImpl::Write(const void* src) { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); - if (ShouldValidateFlagValue(flags_internal::FastTypeId(op_))) { - std::unique_ptr<void, DynValueDeleter> obj{flags_internal::Clone(op_, src), - DynValueDeleter{op_}}; + if (ShouldValidateFlagValue(flags_internal::FastTypeId(op()))) { + std::unique_ptr<void, DynValueDeleter> obj{flags_internal::Clone(op(), src), + DynValueDeleter{op()}}; std::string ignored_error; - std::string src_as_str = flags_internal::Unparse(op_, src); - if (!flags_internal::Parse(op_, src_as_str, obj.get(), &ignored_error)) { + std::string src_as_str = flags_internal::Unparse(op(), src); + if (!flags_internal::Parse(op(), src_as_str, obj.get(), &ignored_error)) { ABSL_INTERNAL_LOG(ERROR, absl::StrCat("Attempt to set flag '", Name(), "' to invalid value ", src_as_str)); } @@ -676,7 +1002,7 @@ // The mode is selected based on 'set_mode' parameter. bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode, ValueSource source, std::string& err) { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); switch (set_mode) { case SET_FLAGS_VALUE: { @@ -687,13 +1013,15 @@ StoreValue(tentative_value.get(), source); if (source == kCommandLine) { - on_command_line_ = true; + MuState s = mu_state_.load(std::memory_order_relaxed); + s.on_command_line = true; + mu_state_.store(s, std::memory_order_release); } break; } case SET_FLAG_IF_DEFAULT: { // set the flag's value, but only if it hasn't been set by someone else - if (modified_) { + if (mu_state_.load(std::memory_order_relaxed).modified) { // TODO(rogeeff): review and fix this semantic. Currently we do not fail // in this case if flag is modified. This is misleading since the flag's // value is not updated even though we return true. @@ -712,19 +1040,23 @@ auto tentative_value = TryParse(value, err); if (!tentative_value) return false; - if (DefaultKind() == FlagDefaultKind::kDynamicValue) { - void* old_value = default_value_.dynamic_value; - default_value_.dynamic_value = tentative_value.release(); + void* new_default = tentative_value.release(); + void* old_value = SetDynamicDefault(this, new_default); + MuState s = mu_state_.load(std::memory_order_relaxed); + const bool was_modified = s.modified; + if (s.DefaultKind() == FlagDefaultKind::kDynamicValue) { tentative_value.reset(old_value); } else { - default_value_.dynamic_value = tentative_value.release(); - def_kind_ = static_cast<uint8_t>(FlagDefaultKind::kDynamicValue); + s.def_kind = static_cast<uint8_t>(FlagDefaultKind::kDynamicValue); + mu_state_.store(s, std::memory_order_release); } - if (!modified_) { + if (!was_modified) { // Need to set both default value *and* current, in this case. - StoreValue(default_value_.dynamic_value, source); - modified_ = false; + StoreValue(new_default, source); + MuState s2 = mu_state_.load(std::memory_order_relaxed); + s2.modified = false; + mu_state_.store(s2, std::memory_order_release); } break; } @@ -736,11 +1068,11 @@ void FlagImpl::CheckDefaultValueParsingRoundtrip() const { std::string v = DefaultValue(); - absl::MutexLock lock(DataGuard()); + absl::MutexLock lock(*DataGuard()); auto dst = MakeInitValue(); std::string error; - if (!flags_internal::Parse(op_, v, dst.get(), &error)) { + if (!flags_internal::Parse(op(), v, dst.get(), &error)) { ABSL_INTERNAL_LOG( FATAL, absl::StrCat("Flag ", Name(), " (from ", Filename(), @@ -753,11 +1085,11 @@ } bool FlagImpl::ValidateInputValue(absl::string_view value) const { - absl::MutexLock l(DataGuard()); + absl::MutexLock l(*DataGuard()); auto obj = MakeInitValue(); std::string ignored_error; - return flags_internal::Parse(op_, value, obj.get(), &ignored_error); + return flags_internal::Parse(op(), value, obj.get(), &ignored_error); } } // namespace flags_internal
diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index 8baff43..3350ed7 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h
@@ -22,10 +22,12 @@ #include <atomic> #include <cstring> #include <memory> +#include <optional> #include <string> #include <type_traits> #include <typeinfo> #include <utility> +#include <vector> #include "absl/base/attributes.h" #include "absl/base/call_once.h" @@ -44,9 +46,12 @@ #include "absl/synchronization/mutex.h" #include "absl/utility/utility.h" - namespace absl { +namespace absl { ABSL_NAMESPACE_BEGIN +class Duration; +class Time; + /////////////////////////////////////////////////////////////////////////////// // Forward declaration of absl::Flag<T> public API. namespace flags_internal { @@ -86,6 +91,7 @@ kParse, kUnparse, kValueOffset, + kTypeName, }; using FlagOpFn = void* (*)(FlagOp, const void*, void*, void*); @@ -154,6 +160,29 @@ return static_cast<ptrdiff_t>(reinterpret_cast<intptr_t>( op(FlagOp::kValueOffset, nullptr, nullptr, nullptr))); } +// Returns string representation of the flag's value type. +inline absl::string_view TypeName(FlagOpFn op) { + absl::string_view result; + op(FlagOp::kTypeName, nullptr, &result, nullptr); + return result; +} + +template <typename T> +constexpr absl::string_view GenTypeName() { +#if ABSL_FLAGS_STRIP_NAMES + return ""; +#elif defined(__clang__) || defined(__GNUC__) + absl::string_view name = __PRETTY_FUNCTION__; + constexpr absl::string_view kKey = "T = "; + const size_t start = name.find(kKey); + if (start == absl::string_view::npos) return ""; + name.remove_prefix(start + kKey.size()); + const size_t end = name.find_first_of(";]"); + return end == absl::string_view::npos ? name : name.substr(0, end); +#else + return ""; +#endif +} // Returns an address of RTTI's typeid(T). template <typename T> @@ -260,7 +289,8 @@ enum class FlagDefaultKind : uint8_t { kDynamicValue = 0, kGenFunc = 1, - kOneWord = 2 // for default values UP to one word in size + kOneWord = 2, // inline default value in descriptor + kDefaultConstructed = 3, // all-zero bytes or default-constructed empty value }; struct FlagDefaultArg { @@ -283,6 +313,110 @@ return T{}; } +template <typename T> +struct IsStdOptional : std::false_type {}; +template <typename U> +struct IsStdOptional<std::optional<U>> : std::true_type {}; + +struct FlagStringDefault { + bool is_literal = false; + absl::string_view sv{}; + + constexpr FlagStringDefault() : is_literal(true), sv() {} + constexpr FlagStringDefault(const char* s) // NOLINT + : is_literal(s != nullptr), + sv(s == nullptr ? absl::string_view() : absl::string_view(s)) {} + template < + typename SV, + std::enable_if_t<std::is_same_v<std::decay_t<SV>, absl::string_view>, + int> = 0> + constexpr FlagStringDefault(SV s) // NOLINT + : is_literal(true), sv(s) {} + template <typename Arg, + std::enable_if_t< + !std::is_convertible_v<const Arg&, const char*> && + !std::is_same_v<std::decay_t<Arg>, absl::string_view> && + !std::is_same_v<std::decay_t<Arg>, FlagStringDefault>, + int> = 0> + constexpr FlagStringDefault(const Arg&) // NOLINT + : is_literal(false), sv() {} +}; + +template <typename T, + std::enable_if_t<!std::is_same_v<T, std::string>, int> = 0> +constexpr bool IsEmptyDefaultArg(EmptyBraces) { + return std::is_same_v<T, std::vector<std::string>> || IsStdOptional<T>::value; +} +template <typename T, std::enable_if_t<!std::is_same_v<T, std::string> && + !IsStdOptional<T>::value, + int> = 0> +constexpr bool IsEmptyDefaultArg(const T&) { + return false; +} +template <typename T, std::enable_if_t<IsStdOptional<T>::value, int> = 0> +constexpr bool IsEmptyDefaultArg(std::nullopt_t) { + return true; +} +template < + typename T, typename Arg, + std::enable_if_t<IsStdOptional<T>::value && + !std::is_same_v<std::decay_t<Arg>, std::nullopt_t>, + int> = 0> +constexpr bool IsEmptyDefaultArg(const Arg&) { + return false; +} +template <typename T, std::enable_if_t<std::is_same_v<T, std::string>, int> = 0> +constexpr bool IsEmptyDefaultArg(FlagStringDefault d) { + return d.is_literal && d.sv.empty(); +} + +template <size_t N> +struct FlagBytes { + uint8_t b[N]; + constexpr bool IsAllZero() const { + for (size_t i = 0; i < N; ++i) { + if (b[i] != 0) return false; + } + return true; + } +}; + +#if ABSL_HAVE_BUILTIN(__builtin_bit_cast) +#define ABSL_FLAGS_INTERNAL_HAS_CONSTEXPR_BIT_CAST 1 +#else +#define ABSL_FLAGS_INTERNAL_HAS_CONSTEXPR_BIT_CAST 0 +#endif + +template <typename T> +struct IsBitCastableDefaultType + : std::bool_constant< +#if ABSL_FLAGS_INTERNAL_HAS_CONSTEXPR_BIT_CAST + std::is_trivially_copyable_v<T> && + (std::is_arithmetic_v<T> || std::is_enum_v<T> || + std::is_same_v<T, absl::int128> || + std::is_same_v<T, absl::uint128> || + std::is_same_v<T, absl::Duration> || std::is_same_v<T, absl::Time>) +#else + false +#endif + > { +}; + +template <typename T> +constexpr FlagBytes<sizeof(T)> BitCastToFlagBytes(const T& v) { +#if ABSL_FLAGS_INTERNAL_HAS_CONSTEXPR_BIT_CAST + if constexpr (IsBitCastableDefaultType<T>::value) { + return __builtin_bit_cast(FlagBytes<sizeof(T)>, v); + } else { + (void)v; + return FlagBytes<sizeof(T)>{}; + } +#else + (void)v; + return FlagBytes<sizeof(T)>{}; +#endif +} + template <typename ValueT, typename GenT, std::enable_if_t<std::is_arithmetic_v<ValueT>, int> = ((void)GenT{}, 0)> @@ -317,13 +451,15 @@ kOneWordAtomic = 1, kSequenceLocked = 2, kHeapAllocated = 3, + kInlineBool = 4, }; // This constexpr function returns the storage kind for the given flag value // type. template <typename T> static constexpr FlagValueStorageKind StorageKind() { - return FlagUseValueAndInitBitStorage<T>::value + return std::is_same_v<T, bool> ? FlagValueStorageKind::kInlineBool + : FlagUseValueAndInitBitStorage<T>::value ? FlagValueStorageKind::kValueAndInitBit : FlagUseOneWordStorage<T>::value ? FlagValueStorageKind::kOneWordAtomic @@ -437,7 +573,7 @@ template <typename T> struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue { constexpr FlagValue() : FlagOneWordValue(0) {} - bool Get(const SequenceLock&, T& dst) const { + bool Get(T& dst) const { int64_t storage = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(storage == 0)) { // This assert is to ensure that the initialization inside FlagImpl::Init @@ -458,7 +594,7 @@ template <typename T> struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { constexpr FlagValue() : FlagOneWordValue() {} - bool Get(const SequenceLock&, T& dst) const { + bool Get(T& dst) const { int64_t one_word_val = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(one_word_val == FlagOneWordValue::Uninitialized())) { return false; @@ -476,15 +612,30 @@ // value once it is initialized. template <typename T> struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> { - bool Get(const SequenceLock& lock, T& dst) const { - return lock.TryRead(&dst, value_words, sizeof(T)); + constexpr FlagValue() : FlagValue(std::make_index_sequence<kNumWords>{}) {} + + bool Get(T& dst) const { + // FlagImpl::SeqLock() computes `&seq_lock` via `AtomicBufferValue() + + // kNumWords`, so verify no padding or fields sit between `value_words` + // and `seq_lock`. + static_assert( + offsetof(FlagValue, seq_lock) - offsetof(FlagValue, value_words) == + kNumWords * sizeof(std::atomic<uint64_t>), + "Unexpected memory layout of sequence-locked FlagValue"); + return seq_lock.TryRead(&dst, value_words, sizeof(T)); } - static constexpr int kNumWords = + static constexpr size_t kNumWords = flags_internal::AlignUp(sizeof(T), sizeof(uint64_t)) / sizeof(uint64_t); alignas(T) alignas( std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords]; + flags_internal::SequenceLock seq_lock; + + private: + template <size_t... Is> + constexpr explicit FlagValue(std::index_sequence<Is...>) + : value_words{(static_cast<void>(Is), uint64_t{0})...} {} }; // This specialization represents the storage of flag values types with the @@ -531,7 +682,7 @@ // ensure Init is done before any access to the flag value. constexpr FlagValue() : FlagMaskedPointerValue(&buffer[0]) {} - bool Get(const SequenceLock&, T& dst) const { + bool Get(T& dst) const { MaskedPointer ptr_value = value.load(std::memory_order_acquire); if (ABSL_PREDICT_TRUE(ptr_value.AllowsUnprotectedRead())) { @@ -545,6 +696,10 @@ T) char buffer[sizeof(T)]{}; }; +// Specialization for bool flags whose value is stored inline in FlagImpl. +template <typename T> +struct FlagValue<T, FlagValueStorageKind::kInlineBool> {}; + /////////////////////////////////////////////////////////////////////////////// // Flag callback auxiliary structs. @@ -569,6 +724,479 @@ FlagOpFn op; }; +inline constexpr uint8_t kNoBuiltinOp = 0xFF; + +template <typename T> +struct BuiltinFlagOpIdImpl { + static constexpr uint8_t kValue = kNoBuiltinOp; +}; +template <> +struct BuiltinFlagOpIdImpl<bool> { + static constexpr uint8_t kValue = 0; +}; +template <> +struct BuiltinFlagOpIdImpl<short> { // NOLINT + static constexpr uint8_t kValue = 1; +}; +template <> +struct BuiltinFlagOpIdImpl<unsigned short> { // NOLINT + static constexpr uint8_t kValue = 2; +}; +template <> +struct BuiltinFlagOpIdImpl<int> { + static constexpr uint8_t kValue = 3; +}; +template <> +struct BuiltinFlagOpIdImpl<unsigned int> { + static constexpr uint8_t kValue = 4; +}; +template <> +struct BuiltinFlagOpIdImpl<long> { // NOLINT + static constexpr uint8_t kValue = 5; +}; +template <> +struct BuiltinFlagOpIdImpl<unsigned long> { // NOLINT + static constexpr uint8_t kValue = 6; +}; +template <> +struct BuiltinFlagOpIdImpl<long long> { // NOLINT + static constexpr uint8_t kValue = 7; +}; +template <> +struct BuiltinFlagOpIdImpl<unsigned long long> { // NOLINT + static constexpr uint8_t kValue = 8; +}; +template <> +struct BuiltinFlagOpIdImpl<float> { + static constexpr uint8_t kValue = 9; +}; +template <> +struct BuiltinFlagOpIdImpl<double> { + static constexpr uint8_t kValue = 10; +}; +template <> +struct BuiltinFlagOpIdImpl<std::string> { + static constexpr uint8_t kValue = 11; +}; +template <> +struct BuiltinFlagOpIdImpl<std::vector<std::string>> { + static constexpr uint8_t kValue = 12; +}; +template <> +struct BuiltinFlagOpIdImpl<absl::int128> { + static constexpr uint8_t kValue = 13; +}; +template <> +struct BuiltinFlagOpIdImpl<absl::uint128> { + static constexpr uint8_t kValue = 14; +}; +template <> +struct BuiltinFlagOpIdImpl<absl::LogSeverity> { + static constexpr uint8_t kValue = 15; +}; +template <> +struct BuiltinFlagOpIdImpl<absl::Duration> { + static constexpr uint8_t kValue = 23; +}; +template <> +struct BuiltinFlagOpIdImpl<absl::Time> { + static constexpr uint8_t kValue = 24; +}; +template <typename U> +struct BuiltinFlagOpIdImpl<std::optional<U>> { + static constexpr uint8_t kValue = static_cast<uint8_t>( + std::is_same_v<U, bool> ? 16 + : std::is_same_v<U, int> ? 17 + : std::is_same_v<U, long> ? 18 // NOLINT + : std::is_same_v<U, long long> ? 19 // NOLINT + : std::is_same_v<U, unsigned long> ? 20 // NOLINT + : std::is_same_v<U, double> ? 21 + : std::is_same_v<U, std::string> ? 22 + : std::is_same_v<U, absl::Duration> ? 25 + : kNoBuiltinOp); +}; + +template <typename T> +constexpr uint8_t BuiltinFlagOpId() { + return BuiltinFlagOpIdImpl<T>::kValue; +} + +enum class FlagMetaKind : uint8_t { + kPtrMeta = 0, + kZeroFlatMeta = 1, + kOneWordFlatMeta = 2, + kTwoWordFlatMeta = 3, +}; + +struct FlagMetaHeader { + constexpr static uint8_t PackKinds(FlagValueStorageKind vk, FlagHelpKind hk, + FlagDefaultKind dk, FlagMetaKind mk) { + return static_cast<uint8_t>((static_cast<uint8_t>(vk) & 0x7u) | + ((static_cast<uint8_t>(hk) & 0x1u) << 3) | + ((static_cast<uint8_t>(dk) & 0x3u) << 4) | + ((static_cast<uint8_t>(mk) & 0x3u) << 6)); + } + + uint8_t kinds; + uint8_t builtin_op_id; + uint8_t name_len; +}; + +constexpr size_t ConstexprStrLen(const char* s) { + size_t len = 0; + if (s != nullptr) { + while (s[len] != '\0') ++len; + } + return len; +} + +template <size_t N> +constexpr const char* ColdHelpPtr(const FixedCharArray<N>& arr) { + return arr.value; +} +constexpr const char* ColdHelpPtr(std::false_type) { return nullptr; } + +// Pointer-based descriptor for built-in types (32 B, omits FlagOpFn pointer). +struct BuiltinPtrFlagMeta { + constexpr BuiltinPtrFlagMeta(FlagMetaHeader hdr_arg, const char* name_arg, + FlagHelpMsg help_arg, + FlagDefaultSrc default_value_arg) + : hdr(hdr_arg), + name(name_arg), + help(help_arg), + default_value(default_value_arg) {} + + FlagMetaHeader hdr; + const char* const name; + const FlagHelpMsg help; + const FlagDefaultSrc default_value; +}; + +// Full pointer-based descriptor for custom/enum types (40 B). +struct FlagMeta { + constexpr FlagMeta(FlagMetaHeader hdr_arg, const char* name_arg, + FlagHelpMsg help_arg, FlagDefaultSrc default_value_arg, + FlagOpFn op_arg) + : hdr(hdr_arg), + name(name_arg), + help(help_arg), + default_value(default_value_arg), + op(op_arg) {} + + FlagMetaHeader hdr; + // Flag's name passed to ABSL_FLAG as second arg. + const char* const name; + // Help message literal or function to generate it. + const FlagHelpMsg help; + // Initial default value source specified in ABSL_FLAG. + const FlagDefaultSrc default_value; + // Type-specific operations vtable. + const FlagOpFn op; +}; + +template <typename T> +constexpr FlagMeta MakeFlagMeta(const char* name, FlagHelpArg help, + FlagDefaultArg default_arg) { + return FlagMeta( + FlagMetaHeader{ + FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(), help.kind, + default_arg.kind, FlagMetaKind::kPtrMeta), + BuiltinFlagOpId<T>(), + static_cast<uint8_t>( + ConstexprStrLen(name) <= 254 ? ConstexprStrLen(name) : 254)}, + name, help.source, default_arg.source, &FlagOps<T>); +} + +template <typename T> +constexpr BuiltinPtrFlagMeta MakeBuiltinPtrFlagMeta( + const char* name, FlagHelpArg help, FlagDefaultArg default_arg) { + return BuiltinPtrFlagMeta( + FlagMetaHeader{ + FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(), help.kind, + default_arg.kind, FlagMetaKind::kPtrMeta), + BuiltinFlagOpId<T>(), + static_cast<uint8_t>( + ConstexprStrLen(name) <= 254 ? ConstexprStrLen(name) : 254)}, + name, help.source, default_arg.source); +} + +// Pointer-free .rodata descriptors for common cases (alignof == 1, 0 pointers, +// 0 relocations in PIE; help stays in flags_help_cold via FlagRegistrar). +template <size_t NameLen> +struct ZeroFlatFlagMeta { + FlagMetaHeader hdr; + char name[NameLen]; +}; + +template <size_t NameLen> +struct OneWordFlatFlagMeta { + FlagMetaHeader hdr; + uint8_t default_bytes[8]; + char name[NameLen]; +}; + +template <size_t NameLen> +struct TwoWordFlatFlagMeta { + FlagMetaHeader hdr; + uint8_t default_bytes[16]; + char name[NameLen]; +}; + +template <typename T, size_t NameLen, size_t... NameI> +constexpr ZeroFlatFlagMeta<NameLen> MakeZeroFlatFlagMeta( + const char* name, FlagDefaultKind def_kind, std::index_sequence<NameI...>) { + return ZeroFlatFlagMeta<NameLen>{ + FlagMetaHeader{FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(), + FlagHelpKind::kLiteral, def_kind, + FlagMetaKind::kZeroFlatMeta), + BuiltinFlagOpId<T>(), static_cast<uint8_t>(NameLen - 1)}, + {name[NameI]..., '\0'}}; +} + +template <typename T, size_t NameLen, size_t... NameI> +constexpr OneWordFlatFlagMeta<NameLen> MakeOneWordFlatFlagMeta( + const char* name, const FlagBytes<sizeof(T)>& def_bytes, + std::index_sequence<NameI...>) { + OneWordFlatFlagMeta<NameLen> res{ + FlagMetaHeader{FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(), + FlagHelpKind::kLiteral, + FlagDefaultKind::kOneWord, + FlagMetaKind::kOneWordFlatMeta), + BuiltinFlagOpId<T>(), static_cast<uint8_t>(NameLen - 1)}, + {}, + {name[NameI]..., '\0'}}; + for (size_t i = 0; i < (sizeof(T) <= 8 ? sizeof(T) : 8); ++i) { + res.default_bytes[i] = def_bytes.b[i]; + } + return res; +} + +template <typename T, size_t NameLen, size_t... NameI> +constexpr TwoWordFlatFlagMeta<NameLen> MakeTwoWordFlatFlagMeta( + const char* name, const FlagBytes<sizeof(T)>& def_bytes, + std::index_sequence<NameI...>) { + TwoWordFlatFlagMeta<NameLen> res{ + FlagMetaHeader{FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(), + FlagHelpKind::kLiteral, + FlagDefaultKind::kOneWord, + FlagMetaKind::kTwoWordFlatMeta), + BuiltinFlagOpId<T>(), static_cast<uint8_t>(NameLen - 1)}, + {}, + {name[NameI]..., '\0'}}; + for (size_t i = 0; i < (sizeof(T) <= 16 ? sizeof(T) : 16); ++i) { + res.default_bytes[i] = def_bytes.b[i]; + } + return res; +} + +template <typename GenT, int = ((void)GenT{}, 0)> +constexpr bool HasConstexprDefault(int) { + return true; +} +template <typename GenT> +constexpr bool HasConstexprDefault(char) { + return false; +} + +template <typename GenT, bool B = GenT::IsEmpty()> +constexpr bool IsConstexprEmptyDefault(int) { + return B; +} +template <typename GenT> +constexpr bool IsConstexprEmptyDefault(char) { + return false; +} + +template <typename ValueT, typename GenT, + bool B = BitCastToFlagBytes<ValueT>(GenT{}.value).IsAllZero()> +constexpr bool IsZeroBytesValue(int) { + return B; +} +template <typename ValueT, typename GenT> +constexpr bool IsZeroBytesValue(char) { + return false; +} + +template <typename ValueT> +constexpr bool IsZeroBytesDefaultConstructed() { + if constexpr (IsBitCastableDefaultType<ValueT>::value) { + return BitCastToFlagBytes<ValueT>(ValueT{}).IsAllZero(); + } else { + return false; + } +} + +enum class FlagMetaShape { + kCustomPtr, + kBuiltinPtr, + kZeroDefaultConstructed, + kZeroBoolTrue, + kOneWordFlat, + kTwoWordFlat, +}; + +template <typename ValueT, typename GenT, + bool IsBitCastable = IsBitCastableDefaultType<ValueT>::value, + bool HasConstexpr = + IsBitCastable ? HasConstexprDefault<GenT>(0) : false> +struct BitCastDefaultShape { + static constexpr FlagMetaShape Value() { return FlagMetaShape::kBuiltinPtr; } +}; + +template <typename ValueT, typename GenT> +struct BitCastDefaultShape<ValueT, GenT, true, true> { + static constexpr FlagMetaShape Value() { + if constexpr (IsZeroBytesValue<ValueT, GenT>(0) && + IsZeroBytesDefaultConstructed<ValueT>()) { + return FlagMetaShape::kZeroDefaultConstructed; + } else if constexpr (!IsZeroBytesValue<ValueT, GenT>(0) && + std::is_same_v<ValueT, bool>) { + return FlagMetaShape::kZeroBoolTrue; + } else if constexpr (!IsZeroBytesValue<ValueT, GenT>(0) && + sizeof(ValueT) <= 8) { + return FlagMetaShape::kOneWordFlat; + } else if constexpr (!IsZeroBytesValue<ValueT, GenT>(0) && + sizeof(ValueT) > 8 && sizeof(ValueT) <= 16) { + return FlagMetaShape::kTwoWordFlat; + } else { + return FlagMetaShape::kBuiltinPtr; + } + } +}; + +template <typename ValueT, typename GenT, size_t NameLen> +constexpr FlagMetaShape SelectLiteralHelpMetaShape() { + if constexpr (BuiltinFlagOpId<ValueT>() == kNoBuiltinOp) { + return FlagMetaShape::kCustomPtr; + } else if constexpr (NameLen > 255) { + return FlagMetaShape::kBuiltinPtr; + } else if constexpr (IsBitCastableDefaultType<ValueT>::value) { + return BitCastDefaultShape<ValueT, GenT>::Value(); + } else if constexpr (std::is_same_v<ValueT, std::string> || + std::is_same_v<ValueT, std::vector<std::string>> || + IsStdOptional<ValueT>::value) { + if constexpr (IsConstexprEmptyDefault<GenT>(0)) { + return FlagMetaShape::kZeroDefaultConstructed; + } else { + return FlagMetaShape::kBuiltinPtr; + } + } else { + return FlagMetaShape::kBuiltinPtr; + } +} + +template <FlagMetaShape Shape, size_t NameLen, typename GenT> +struct FlagMetaMaker; + +template <size_t NameLen, typename GenT> +struct FlagMetaMaker<FlagMetaShape::kCustomPtr, NameLen, GenT> { + using type = FlagMeta; + template <typename ValueT, typename HelpGenT, typename HelpStorageT> + static constexpr FlagMeta Make(const char* name, + const HelpStorageT& help_storage) { + return MakeFlagMeta<ValueT>(name, HelpArg<HelpGenT>(help_storage), + DefaultArg<ValueT, GenT>(0)); + } +}; + +template <size_t NameLen, typename GenT> +struct FlagMetaMaker<FlagMetaShape::kBuiltinPtr, NameLen, GenT> { + using type = BuiltinPtrFlagMeta; + template <typename ValueT, typename HelpGenT, typename HelpStorageT> + static constexpr BuiltinPtrFlagMeta Make(const char* name, + const HelpStorageT& help_storage) { + return MakeBuiltinPtrFlagMeta<ValueT>(name, HelpArg<HelpGenT>(help_storage), + DefaultArg<ValueT, GenT>(0)); + } +}; + +template <size_t NameLen, typename GenT> +struct FlagMetaMaker<FlagMetaShape::kZeroDefaultConstructed, NameLen, GenT> { + using type = ZeroFlatFlagMeta<NameLen>; + template <typename ValueT, typename HelpGenT, typename HelpStorageT> + static constexpr ZeroFlatFlagMeta<NameLen> Make(const char* name, + const HelpStorageT&) { + return MakeZeroFlatFlagMeta<ValueT, NameLen>( + name, FlagDefaultKind::kDefaultConstructed, + std::make_index_sequence<NameLen - 1>{}); + } +}; + +template <size_t NameLen, typename GenT> +struct FlagMetaMaker<FlagMetaShape::kZeroBoolTrue, NameLen, GenT> { + using type = ZeroFlatFlagMeta<NameLen>; + template <typename ValueT, typename HelpGenT, typename HelpStorageT> + static constexpr ZeroFlatFlagMeta<NameLen> Make(const char* name, + const HelpStorageT&) { + return MakeZeroFlatFlagMeta<ValueT, NameLen>( + name, FlagDefaultKind::kOneWord, + std::make_index_sequence<NameLen - 1>{}); + } +}; + +template <size_t NameLen, typename GenT> +struct FlagMetaMaker<FlagMetaShape::kOneWordFlat, NameLen, GenT> { + using type = OneWordFlatFlagMeta<NameLen>; + template <typename ValueT, typename HelpGenT, typename HelpStorageT> + static constexpr OneWordFlatFlagMeta<NameLen> Make(const char* name, + const HelpStorageT&) { + return MakeOneWordFlatFlagMeta<ValueT, NameLen>( + name, BitCastToFlagBytes<ValueT>(GenT{}.value), + std::make_index_sequence<NameLen - 1>{}); + } +}; + +template <size_t NameLen, typename GenT> +struct FlagMetaMaker<FlagMetaShape::kTwoWordFlat, NameLen, GenT> { + using type = TwoWordFlatFlagMeta<NameLen>; + template <typename ValueT, typename HelpGenT, typename HelpStorageT> + static constexpr TwoWordFlatFlagMeta<NameLen> Make(const char* name, + const HelpStorageT&) { + return MakeTwoWordFlatFlagMeta<ValueT, NameLen>( + name, BitCastToFlagBytes<ValueT>(GenT{}.value), + std::make_index_sequence<NameLen - 1>{}); + } +}; + +template <typename ValueT, typename GenT, typename HelpGenT, size_t NameLen, + typename HelpStorageT> +struct FlagMetaSelector; + +template <typename ValueT, typename GenT, typename HelpGenT, size_t NameLen, + size_t HelpLen> +struct FlagMetaSelector<ValueT, GenT, HelpGenT, NameLen, + FixedCharArray<HelpLen>> { + using Maker = + FlagMetaMaker<SelectLiteralHelpMetaShape<ValueT, GenT, NameLen>(), + NameLen, GenT>; + using type = typename Maker::type; + static constexpr type Make(const char* name, + const FixedCharArray<HelpLen>& help_storage) { + return Maker::template Make<ValueT, HelpGenT>(name, help_storage); + } +}; + +template <typename ValueT, typename GenT, typename HelpGenT, size_t NameLen> +struct FlagMetaSelector<ValueT, GenT, HelpGenT, NameLen, std::false_type> { + using Maker = FlagMetaMaker<BuiltinFlagOpId<ValueT>() != kNoBuiltinOp + ? FlagMetaShape::kBuiltinPtr + : FlagMetaShape::kCustomPtr, + NameLen, GenT>; + using type = typename Maker::type; + static constexpr type Make(const char* name, std::false_type help_storage) { + return Maker::template Make<ValueT, HelpGenT>(name, help_storage); + } +}; + +template <typename ValueT, typename GenT, typename HelpGenT, size_t NameLen, + typename HelpStorageT> +constexpr typename FlagMetaSelector<ValueT, GenT, HelpGenT, NameLen, + HelpStorageT>::type +SelectFlagMeta(const char* name, const HelpStorageT& help_storage, int) { + return FlagMetaSelector<ValueT, GenT, HelpGenT, NameLen, HelpStorageT>::Make( + name, help_storage); +} + class FlagState; // These are only used as constexpr global objects. @@ -579,37 +1207,62 @@ #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #endif class FlagImpl final : public CommandLineFlag { + private: + static constexpr uint8_t kMaxInlineModCount = 7; + + struct MuState { + uint8_t def_kind : 2; + bool modified : 1; + bool on_command_line : 1; + bool has_callback : 1; + uint8_t mod_count : 3; + + constexpr FlagDefaultKind DefaultKind() const { + return static_cast<FlagDefaultKind>(def_kind); + } + }; + static_assert(sizeof(MuState) == 1, "MuState must be 1 byte"); + static_assert(std::atomic<MuState>::is_always_lock_free, + "std::atomic<MuState> must be lock-free"); + public: - constexpr FlagImpl(const char* name, const char* type_name, - const char* filename, FlagOpFn op, FlagHelpArg help, - FlagValueStorageKind value_kind, - FlagDefaultArg default_arg) - : name_(name), - type_name_(type_name), - filename_(filename), - op_(op), - help_(help.source), - help_source_kind_(static_cast<uint8_t>(help.kind)), - value_storage_kind_(static_cast<uint8_t>(value_kind)), - def_kind_(static_cast<uint8_t>(default_arg.kind)), - modified_(false), - on_command_line_(false), - callback_(nullptr), - default_value_(default_arg.source), - data_guard_{} {} + constexpr explicit FlagImpl(const FlagMetaHeader* meta) + : meta_(meta), + init_control_{}, + file_id_(0), + mu_state_(MuState{static_cast<uint8_t>((meta->kinds >> 4) & 0x3u), + false, false, false, 0}), + inline_bool_value_(0) {} + + constexpr FlagImpl(const FlagMeta* meta, FlagHelpKind /*help_kind*/, + FlagValueStorageKind /*value_kind*/, + FlagDefaultKind default_kind) + : meta_(&meta->hdr), + init_control_{}, + file_id_(0), + mu_state_(MuState{static_cast<uint8_t>(default_kind), false, false, + false, 0}), + inline_bool_value_(0) {} + + bool TryGetInlineBool(bool& dst) const { + uint8_t s = inline_bool_value_.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(s == 0)) return false; + dst = static_cast<bool>(s & 1u); + return true; + } // Constant access methods - int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(DataGuard()); - bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(DataGuard()); - void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(DataGuard()); - void Read(bool* value) const ABSL_LOCKS_EXCLUDED(DataGuard()) { + int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(*DataGuard()); + bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(*DataGuard()); + void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + void Read(bool* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { *value = ReadOneBool(); } template <typename T, std::enable_if_t<flags_internal::StorageKind<T>() == FlagValueStorageKind::kOneWordAtomic, int> = 0> - void Read(T* value) const ABSL_LOCKS_EXCLUDED(DataGuard()) { + void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { int64_t v = ReadOneWord(); std::memcpy(value, static_cast<const void*>(&v), sizeof(T)); } @@ -617,17 +1270,19 @@ std::enable_if_t<flags_internal::StorageKind<T>() == FlagValueStorageKind::kValueAndInitBit, int> = 0> - void Read(T* value) const ABSL_LOCKS_EXCLUDED(DataGuard()) { + void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { *value = absl::bit_cast<FlagValueAndInitBit<T>>(ReadOneWord()).value; } // Mutating access methods - void Write(const void* src) ABSL_LOCKS_EXCLUDED(DataGuard()); + void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard()); + void SetFilename(const char* filename); + void SetColdHelp(const char* cold_help); // Interfaces to operate on callbacks. void SetCallback(const FlagCallbackFunc mutation_callback) - ABSL_LOCKS_EXCLUDED(DataGuard()); - void InvokeCallback() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); + void InvokeCallback() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Used in read/write operations to validate source/target has correct type. // For example if flag is declared as absl::Flag<int> FLAGS_foo, a call to @@ -643,12 +1298,19 @@ friend class Flag; friend class FlagState; + FlagOpFn op() const; + const BuiltinPtrFlagMeta* PtrMeta() const { + return reinterpret_cast<const BuiltinPtrFlagMeta*>(meta_); + } + const void* DefaultValueStoragePtr() const; + const char* DefaultCStringPtr() const; + // Ensures that `data_guard_` is initialized and returns it. - absl::Mutex& DataGuard() const + absl::Mutex* DataGuard() const ABSL_LOCK_RETURNED(reinterpret_cast<absl::Mutex*>(data_guard_)); // Returns heap allocated value of type T initialized with default value. std::unique_ptr<void, DynValueDeleter> MakeInitValue() const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Flag initialization called via absl::call_once. void Init(); @@ -656,7 +1318,7 @@ // respect const correctness, so be very careful using them. // This is a shared helper routine which encapsulates most of the magic. Since - // it is only used inside the three routines below, which are defined in + // it is only used inside the routines below, which are defined in // flag.cc, we can define it in that file as well. template <typename StorageT> StorageT* OffsetValue() const; @@ -664,6 +1326,9 @@ // The same as above, but used for sequencelock-protected storage. std::atomic<uint64_t>* AtomicBufferValue() const; + // Accessor for the SequenceLock inside FlagValue<T, kSequenceLocked>. + flags_internal::SequenceLock& SeqLock() const; + // This is an accessor for a value stored as one word atomic. Returns a // mutable reference to an atomic value. std::atomic<int64_t>& OneWordValue() const; @@ -674,25 +1339,25 @@ // returns new value. Otherwise returns nullptr. std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value, std::string& err) const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Stores the flag value based on the pointer to the source. void StoreValue(const void* src, ValueSource source) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); - // Copy the flag data, protected by `seq_lock_` into `dst`. + // Copy the flag data, protected by `SeqLock()` into `dst`. // // REQUIRES: ValueStorageKind() == kSequenceLocked. - void ReadSequenceLockedData(void* dst) const ABSL_LOCKS_EXCLUDED(DataGuard()); + void ReadSequenceLockedData(void* dst) const + ABSL_LOCKS_EXCLUDED(*DataGuard()); FlagHelpKind HelpSourceKind() const { - return static_cast<FlagHelpKind>(help_source_kind_); + return static_cast<FlagHelpKind>((meta_->kinds >> 3) & 0x1u); } FlagValueStorageKind ValueStorageKind() const { - return static_cast<FlagValueStorageKind>(value_storage_kind_); + return static_cast<FlagValueStorageKind>(meta_->kinds & 0x7u); } - FlagDefaultKind DefaultKind() const - ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()) { - return static_cast<FlagDefaultKind>(def_kind_); + FlagMetaKind MetaKind() const { + return static_cast<FlagMetaKind>((meta_->kinds >> 6) & 0x3u); } // CommandLineFlag interface implementation @@ -702,86 +1367,63 @@ std::string Help() const override; FlagFastTypeId TypeId() const override; bool IsSpecifiedOnCommandLine() const override - ABSL_LOCKS_EXCLUDED(DataGuard()); - std::string DefaultValue() const override ABSL_LOCKS_EXCLUDED(DataGuard()); - std::string CurrentValue() const override ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); + std::string DefaultValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + std::string CurrentValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard()); bool ValidateInputValue(absl::string_view value) const override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); void CheckDefaultValueParsingRoundtrip() const override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); - int64_t ModificationCount() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(DataGuard()); + int64_t ModificationCount() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + // Lock-free check used by SaveState() and RestoreState() to skip lazy + // initialization on flags that have never been mutated. Safe without + // DataGuard() because all mutations update mu_state_ under DataGuard() + // with release semantics after writing the flag value, and observing a + // zero modification count means the flag still holds its initial + // compile-time state. + bool IsUnmodifiedFast() const { + return mu_state_.load(std::memory_order_acquire).mod_count == 0; + } bool CanSkipInitOnSave() const; + void RecordMutation() ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Interfaces to save and restore flags to/from persistent state. // Returns current flag state or nullptr if flag does not support // saving and restoring a state. std::unique_ptr<FlagStateInterface> SaveState() override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); // Restores the flag state to the supplied state object. If there is // nothing to restore returns false. Otherwise returns true. bool RestoreState(const FlagState& flag_state) - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); bool ParseFrom(absl::string_view value, FlagSettingMode set_mode, ValueSource source, std::string& error) override - ABSL_LOCKS_EXCLUDED(DataGuard()); + ABSL_LOCKS_EXCLUDED(*DataGuard()); - // Immutable flag's state. + // Pointer to immutable flag descriptor in .rodata / .data.rel.ro. + const FlagMetaHeader* const meta_; - // Flags name passed to ABSL_FLAG as second arg. - const char* const name_; - - // Flags type passed to ABSL_FLAG as first arg. - const char* const type_name_; - - // The file name where ABSL_FLAG resides. - const char* const filename_; - // Type-specific operations vtable. - const FlagOpFn op_; - // Help message literal or function to generate it. - const FlagHelpMsg help_; - // Indicates if help message was supplied as literal or generator func. - const uint8_t help_source_kind_ : 1; - // Kind of storage this flag is using for the flag's value. - const uint8_t value_storage_kind_ : 2; - - uint8_t : 0; // The bytes containing the const bitfields must not be - // shared with bytes containing the mutable bitfields. - - // Mutable flag's state (guarded by `data_guard_`). - - // def_kind_ is not guard by DataGuard() since it is accessed in Init without - // locks. - uint8_t def_kind_ : 2; - // Has this flag's value been modified? - bool modified_ : 1 ABSL_GUARDED_BY(DataGuard()); - // Has this flag been specified on command line. - bool on_command_line_ : 1 ABSL_GUARDED_BY(DataGuard()); + // Per-flag mutex guard, placement-initialized in Init(). + alignas(sizeof(void*)) mutable char data_guard_[sizeof(absl::Mutex)]{}; // Unique tag for absl::call_once call to initialize this flag. absl::once_flag init_control_; - // Sequence lock / mutation counter. - flags_internal::SequenceLock seq_lock_; + // Deduplicated filename index assigned on registration (0 = unset). + uint16_t file_id_; - // Optional flag's callback and absl::Mutex to guard the invocations. - FlagCallback* callback_ ABSL_GUARDED_BY(DataGuard()); - // Either a pointer to the function generating the default value based on the - // value specified in ABSL_FLAG or pointer to the dynamically set default - // value via SetCommandLineOptionWithMode. def_kind_ is used to distinguish - // these two cases. - FlagDefaultSrc default_value_; + // Atomic state bitfields (def_kind, modified, on_command_line, has_callback, + // and saturating 3-bit mod_count where 7 indicates overflow in side table). + // All writes occur under DataGuard() with release semantics; SaveState() and + // RestoreState() perform lock-free acquire loads to fast-path unmodified + // flags without triggering lazy initialization. + std::atomic<MuState> mu_state_; - // This is reserved space for an absl::Mutex to guard flag data. It will be - // initialized in FlagImpl::Init via placement new. - // We can't use "absl::Mutex data_guard_", since this class is not literal. - // We do not want to use "absl::Mutex* data_guard_", since this would require - // heap allocation during initialization, which is both slows program startup - // and can fail. Using reserved space + placement new allows us to avoid both - // problems. - alignas(absl::Mutex) mutable unsigned char data_guard_[sizeof(absl::Mutex)]; + // Inline atomic storage for Flag<bool> (0 = uninit, 2 = false, 3 = true). + mutable std::atomic<uint8_t> inline_bool_value_; }; #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop @@ -791,25 +1433,52 @@ // The Flag object parameterized by the flag's value type. This class implements // flag reflection handle interface. +template <typename T, + FlagValueStorageKind Kind = flags_internal::StorageKind<T>()> +struct FlagStorage { + constexpr explicit FlagStorage(const FlagMetaHeader* meta) + : impl_(meta), value_() {} + constexpr FlagStorage(const FlagMeta* meta, FlagHelpKind help_kind, + FlagDefaultKind default_kind) + : impl_(meta, help_kind, Kind, default_kind), value_() {} + + // The implementation depends on value_ field to be placed exactly after the + // impl_ field, so that impl_ can figure out the offset to the value and + // access it. + FlagImpl impl_; + FlagValue<T, Kind> value_; +}; + template <typename T> -class Flag { +struct FlagStorage<T, FlagValueStorageKind::kInlineBool> { + constexpr explicit FlagStorage(const FlagMetaHeader* meta) : impl_(meta) {} + constexpr FlagStorage(const FlagMeta* meta, FlagHelpKind help_kind, + FlagDefaultKind default_kind) + : impl_(meta, help_kind, FlagValueStorageKind::kInlineBool, + default_kind) {} + + FlagImpl impl_; +}; + +template <typename T> +class Flag : private FlagStorage<T> { public: - constexpr Flag(const char* name, const char* type_name, const char* filename, - FlagHelpArg help, const FlagDefaultArg default_arg) - : impl_(name, type_name, filename, &FlagOps<T>, help, - flags_internal::StorageKind<T>(), default_arg), - value_() {} + constexpr explicit Flag(const FlagMetaHeader* meta) : FlagStorage<T>(meta) {} + + constexpr Flag(const FlagMeta* meta, FlagHelpKind help_kind, + FlagDefaultKind default_kind) + : FlagStorage<T>(meta, help_kind, default_kind) {} // CommandLineFlag interface - absl::string_view Name() const { return impl_.Name(); } - std::string Filename() const { return impl_.Filename(); } - std::string Help() const { return impl_.Help(); } + absl::string_view Name() const { return this->impl_.Name(); } + std::string Filename() const { return this->impl_.Filename(); } + std::string Help() const { return this->impl_.Help(); } // Do not use. To be removed. bool IsSpecifiedOnCommandLine() const { - return impl_.IsSpecifiedOnCommandLine(); + return this->impl_.IsSpecifiedOnCommandLine(); } - std::string DefaultValue() const { return impl_.DefaultValue(); } - std::string CurrentValue() const { return impl_.CurrentValue(); } + std::string DefaultValue() const { return this->impl_.DefaultValue(); } + std::string CurrentValue() const { return this->impl_.CurrentValue(); } private: template <typename, bool> @@ -826,28 +1495,28 @@ U u; #if !defined(NDEBUG) - impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + this->impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); #endif - if (ABSL_PREDICT_FALSE(!value_.Get(impl_.seq_lock_, u.value))) { - impl_.Read(&u.value); + if constexpr (flags_internal::StorageKind<T>() == + FlagValueStorageKind::kInlineBool) { + if (ABSL_PREDICT_FALSE(!this->impl_.TryGetInlineBool(u.value))) { + this->impl_.Read(&u.value); + } + } else { + if (ABSL_PREDICT_FALSE(!this->value_.Get(u.value))) { + this->impl_.Read(&u.value); + } } return std::move(u.value); } void Set(const T& v) { - impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); - impl_.Write(&v); + this->impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + this->impl_.Write(&v); } // Access to the reflection. - const CommandLineFlag& Reflect() const { return impl_; } - - // Flag's data - // The implementation depends on value_ field to be placed exactly after the - // impl_ field, so that impl_ can figure out the offset to the value and - // access it. - FlagImpl impl_; - FlagValue<T> value_; + const CommandLineFlag& Reflect() const { return this->impl_; } }; /////////////////////////////////////////////////////////////////////////////// @@ -895,6 +1564,12 @@ *static_cast<T*>(v2) = *static_cast<const T*>(v1); return nullptr; case FlagOp::kCopyConstruct: + if constexpr (std::is_default_constructible_v<T>) { + if (v1 == nullptr) { + new (v2) T(); + return nullptr; + } + } new (v2) T(*static_cast<const T*>(v1)); return nullptr; case FlagOp::kSizeof: @@ -925,6 +1600,9 @@ size_t offset = (sizeof(FlagImpl) + round_to - 1) / round_to * round_to; return reinterpret_cast<void*>(offset); } + case FlagOp::kTypeName: + *static_cast<absl::string_view*>(v2) = GenTypeName<T>(); + return nullptr; } return nullptr; } @@ -937,10 +1615,18 @@ template <typename T, bool do_register> class FlagRegistrar { public: - constexpr explicit FlagRegistrar(Flag<T>& flag, const char* filename) + constexpr explicit FlagRegistrar(Flag<T>& flag, const char* filename, + const char* cold_help = nullptr) : flag_(flag) { - if (do_register) + if (filename != nullptr) { + flag_.impl_.SetFilename(filename); + } + if (cold_help != nullptr) { + flag_.impl_.SetColdHelp(cold_help); + } + if (do_register) { flags_internal::RegisterCommandLineFlag(flag_.impl_, filename); + } } FlagRegistrar OnUpdate(FlagCallbackFunc cb) && {
diff --git a/absl/flags/internal/sequence_lock.h b/absl/flags/internal/sequence_lock.h index 1729d7f..f3c95ec 100644 --- a/absl/flags/internal/sequence_lock.h +++ b/absl/flags/internal/sequence_lock.h
@@ -66,11 +66,6 @@ lock_.store(0, std::memory_order_release); } - // Returns true if the flag is either uninitialized or initialized with zero - // modifications. - bool IsUnmodified() const { - return lock_.load(std::memory_order_acquire) <= 0; - } // Copy "size" bytes of data from "src" to "dst", protected as a read-side // critical section of the sequence lock. @@ -124,25 +119,6 @@ lock_.store(orig_seq + 2, std::memory_order_release); } - // Return the number of times that Write() has been called. - // - // REQUIRES: This must be externally synchronized against concurrent calls to - // `Write()` or `IncrementModificationCount()`. - // REQUIRES: `MarkInitialized()` must have been previously called. - int64_t ModificationCount() const { - int64_t val = lock_.load(std::memory_order_relaxed); - assert(val != kUninitialized && (val & 1) == 0); - return val / 2; - } - - // REQUIRES: This must be externally synchronized against concurrent calls to - // `Write()` or `ModificationCount()`. - // REQUIRES: `MarkInitialized()` must have been previously called. - void IncrementModificationCount() { - int64_t val = lock_.load(std::memory_order_relaxed); - assert(val != kUninitialized); - lock_.store(val + 2, std::memory_order_relaxed); - } private: // Perform the equivalent of "memcpy(dst, src, size)", but using relaxed