absl/flags/internal/flag.h

//
// Copyright 2019 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef ABSL_FLAGS_INTERNAL_FLAG_H_
#define ABSL_FLAGS_INTERNAL_FLAG_H_

#include <stddef.h>
#include <stdint.h>

#include <atomic>
#include <cstring>
#include <memory>
#include <optional>
#include <string>
#include <type_traits>
#include <typeinfo>
#include <utility>
#include <vector>

#include "absl/base/attributes.h"
#include "absl/base/call_once.h"
#include "absl/base/casts.h"
#include "absl/base/config.h"
#include "absl/base/optimization.h"
#include "absl/base/thread_annotations.h"
#include "absl/flags/commandlineflag.h"
#include "absl/flags/config.h"
#include "absl/flags/internal/commandlineflag.h"
#include "absl/flags/internal/registry.h"
#include "absl/flags/internal/sequence_lock.h"
#include "absl/flags/marshalling.h"
#include "absl/meta/type_traits.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "absl/utility/utility.h"

namespace absl {
ABSL_NAMESPACE_BEGIN

class Duration;
class Time;

///////////////////////////////////////////////////////////////////////////////
// Forward declaration of absl::Flag<T> public API.
namespace flags_internal {
template <typename T>
class Flag;
}  // namespace flags_internal

template <typename T>
using Flag = flags_internal::Flag<T>;

template <typename T>
[[nodiscard]] T GetFlag(const absl::Flag<T>& flag);

template <typename T>
void SetFlag(absl::Flag<T>* flag, const T& v);

template <typename T, typename V>
void SetFlag(absl::Flag<T>* flag, const V& v);

template <typename U>
const CommandLineFlag& GetFlagReflectionHandle(const absl::Flag<U>& f);

///////////////////////////////////////////////////////////////////////////////
// Flag value type operations, eg., parsing, copying, etc. are provided
// by function specific to that type with a signature matching FlagOpFn.

namespace flags_internal {

enum class FlagOp {
  kAlloc,
  kDelete,
  kCopy,
  kCopyConstruct,
  kSizeof,
  kFastTypeId,
  kRuntimeTypeId,
  kParse,
  kUnparse,
  kValueOffset,
  kTypeName,
};
using FlagOpFn = void* (*)(FlagOp, const void*, void*, void*);

// Forward declaration for Flag value specific operations.
template <typename T>
void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3);

// Allocate aligned memory for a flag value.
inline void* Alloc(FlagOpFn op) {
  return op(FlagOp::kAlloc, nullptr, nullptr, nullptr);
}
// Deletes memory interpreting obj as flag value type pointer.
inline void Delete(FlagOpFn op, void* obj) {
  op(FlagOp::kDelete, nullptr, obj, nullptr);
}
// Copies src to dst interpreting as flag value type pointers.
inline void Copy(FlagOpFn op, const void* src, void* dst) {
  op(FlagOp::kCopy, src, dst, nullptr);
}
// Construct a copy of flag value in a location pointed by dst
// based on src - pointer to the flag's value.
inline void CopyConstruct(FlagOpFn op, const void* src, void* dst) {
  op(FlagOp::kCopyConstruct, src, dst, nullptr);
}
// Makes a copy of flag value pointed by obj.
inline void* Clone(FlagOpFn op, const void* obj) {
  void* res = flags_internal::Alloc(op);
  flags_internal::CopyConstruct(op, obj, res);
  return res;
}
// Returns true if parsing of input text is successful.
inline bool Parse(FlagOpFn op, absl::string_view text, void* dst,
                  std::string* error) {
  return op(FlagOp::kParse, &text, dst, error) != nullptr;
}
// Returns string representing supplied value.
inline std::string Unparse(FlagOpFn op, const void* val) {
  std::string result;
  op(FlagOp::kUnparse, val, &result, nullptr);
  return result;
}
// Returns size of flag value type.
inline size_t Sizeof(FlagOpFn op) {
  // This sequence of casts reverses the sequence from
  // `flags_internal::FlagOps()`
  return static_cast<size_t>(reinterpret_cast<intptr_t>(
      op(FlagOp::kSizeof, nullptr, nullptr, nullptr)));
}
// Returns fast type id corresponding to the value type.
inline FlagFastTypeId FastTypeId(FlagOpFn op) {
  return absl::bit_cast<FlagFastTypeId>(
      op(FlagOp::kFastTypeId, nullptr, nullptr, nullptr));
}
// Returns fast type id corresponding to the value type.
inline const std::type_info* RuntimeTypeId(FlagOpFn op) {
  return reinterpret_cast<const std::type_info*>(
      op(FlagOp::kRuntimeTypeId, nullptr, nullptr, nullptr));
}
// Returns offset of the field value_ from the field impl_ inside of
// absl::Flag<T> data. Given FlagImpl pointer p you can get the
// location of the corresponding value as:
//      reinterpret_cast<char*>(p) + ValueOffset().
inline ptrdiff_t ValueOffset(FlagOpFn op) {
  // This sequence of casts reverses the sequence from
  // `flags_internal::FlagOps()`
  return static_cast<ptrdiff_t>(reinterpret_cast<intptr_t>(
      op(FlagOp::kValueOffset, nullptr, nullptr, nullptr)));
}
// Returns string representation of the flag's value type.
inline absl::string_view TypeName(FlagOpFn op) {
  absl::string_view result;
  op(FlagOp::kTypeName, nullptr, &result, nullptr);
  return result;
}

template <typename T>
constexpr absl::string_view GenTypeName() {
#if ABSL_FLAGS_STRIP_NAMES
  return "";
#elif defined(__clang__) || defined(__GNUC__)
  absl::string_view name = __PRETTY_FUNCTION__;
  constexpr absl::string_view kKey = "T = ";
  const size_t start = name.find(kKey);
  if (start == absl::string_view::npos) return "";
  name.remove_prefix(start + kKey.size());
  const size_t end = name.find_first_of(";]");
  return end == absl::string_view::npos ? name : name.substr(0, end);
#else
  return "";
#endif
}

// Returns an address of RTTI's typeid(T).
template <typename T>
inline const std::type_info* GenRuntimeTypeId() {
#ifdef ABSL_INTERNAL_HAS_RTTI
  return &typeid(T);
#else
  return nullptr;
#endif
}

///////////////////////////////////////////////////////////////////////////////
// Flag help auxiliary structs.

// This is help argument for absl::Flag encapsulating the string literal pointer
// or pointer to function generating it as well as enum discriminating two
// cases.
using HelpGenFunc = std::string (*)();

template <size_t N>
struct FixedCharArray {
  char value[N];

  template <size_t... I>
  static constexpr FixedCharArray<N> FromLiteralString(
      absl::string_view str, std::index_sequence<I...>) {
    return (void)str, FixedCharArray<N>({{str[I]..., '\0'}});
  }
};

template <typename Gen, size_t N = Gen::Value().size()>
constexpr FixedCharArray<N + 1> HelpStringAsArray(int) {
  return FixedCharArray<N + 1>::FromLiteralString(
      Gen::Value(), std::make_index_sequence<N>{});
}

template <typename Gen>
constexpr std::false_type HelpStringAsArray(char) {
  return std::false_type{};
}

union FlagHelpMsg {
  constexpr explicit FlagHelpMsg(const char* help_msg) : literal(help_msg) {}
  constexpr explicit FlagHelpMsg(HelpGenFunc help_gen) : gen_func(help_gen) {}

  const char* literal;
  HelpGenFunc gen_func;
};

enum class FlagHelpKind : uint8_t { kLiteral = 0, kGenFunc = 1 };

struct FlagHelpArg {
  FlagHelpMsg source;
  FlagHelpKind kind;
};

extern const char kStrippedFlagHelp[];

// These two HelpArg overloads allows us to select at compile time one of two
// way to pass Help argument to absl::Flag. We'll be passing
// AbslFlagHelpGenFor##name as Gen and integer 0 as a single argument to prefer
// first overload if possible. If help message is evaluatable on constexpr
// context We'll be able to make FixedCharArray out of it and we'll choose first
// overload. In this case the help message expression is immediately evaluated
// and is used to construct the absl::Flag. No additional code is generated by
// ABSL_FLAG Otherwise SFINAE kicks in and first overload is dropped from the
// consideration, in which case the second overload will be used. The second
// overload does not attempt to evaluate the help message expression
// immediately and instead delays the evaluation by returning the function
// pointer (&T::NonConst) generating the help message when necessary. This is
// evaluatable in constexpr context, but the cost is an extra function being
// generated in the ABSL_FLAG code.
template <typename Gen, size_t N>
constexpr FlagHelpArg HelpArg(const FixedCharArray<N>& value) {
  return {FlagHelpMsg(value.value), FlagHelpKind::kLiteral};
}

template <typename Gen>
constexpr FlagHelpArg HelpArg(std::false_type) {
  return {FlagHelpMsg(&Gen::NonConst), FlagHelpKind::kGenFunc};
}

///////////////////////////////////////////////////////////////////////////////
// Flag default value auxiliary structs.

// Signature for the function generating the initial flag value (usually
// based on default value supplied in flag's definition)
using FlagDfltGenFunc = void (*)(void*);

union FlagDefaultSrc {
  constexpr explicit FlagDefaultSrc(FlagDfltGenFunc gen_func_arg)
      : gen_func(gen_func_arg) {}

#define ABSL_FLAGS_INTERNAL_DFLT_FOR_TYPE(T, name) \
  T name##_value;                                  \
  constexpr explicit FlagDefaultSrc(T value) : name##_value(value) {}  // NOLINT
  ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(ABSL_FLAGS_INTERNAL_DFLT_FOR_TYPE)
#undef ABSL_FLAGS_INTERNAL_DFLT_FOR_TYPE

  void* dynamic_value;
  FlagDfltGenFunc gen_func;
};

enum class FlagDefaultKind : uint8_t {
  kDynamicValue = 0,
  kGenFunc = 1,
  kOneWord = 2,  // inline default value in descriptor
};

struct FlagDefaultArg {
  FlagDefaultSrc source;
  FlagDefaultKind kind;
};

// This struct and corresponding overload to InitDefaultValue are used to
// facilitate usage of {} as default value in ABSL_FLAG macro.
// TODO(rogeeff): Fix handling types with explicit constructors.
struct EmptyBraces {};

template <typename T>
constexpr T InitDefaultValue(T t) {
  return t;
}

template <typename T>
constexpr T InitDefaultValue(EmptyBraces) {
  return T{};
}

template <typename ValueT, typename GenT,
          std::enable_if_t<std::is_arithmetic_v<ValueT>, int> = ((void)GenT{},
                                                                 0)>
constexpr FlagDefaultArg DefaultArg(int) {
  return {FlagDefaultSrc(GenT{}.value), FlagDefaultKind::kOneWord};
}

template <typename ValueT, typename GenT>
constexpr FlagDefaultArg DefaultArg(char) {
  return {FlagDefaultSrc(&GenT::Gen), FlagDefaultKind::kGenFunc};
}

///////////////////////////////////////////////////////////////////////////////
// Flag storage selector traits. Each trait indicates what kind of storage kind
// to use for the flag value.

template <typename T>
using FlagUseValueAndInitBitStorage =
    std::bool_constant<std::is_trivially_copyable_v<T> &&
                       std::is_default_constructible_v<T> && (sizeof(T) < 8)>;

template <typename T>
using FlagUseOneWordStorage =
    std::bool_constant<std::is_trivially_copyable_v<T> && (sizeof(T) <= 8)>;

template <class T>
using FlagUseSequenceLockStorage =
    std::bool_constant<std::is_trivially_copyable_v<T> && (sizeof(T) > 8)>;

enum class FlagValueStorageKind : uint8_t {
  kValueAndInitBit = 0,
  kOneWordAtomic = 1,
  kSequenceLocked = 2,
  kHeapAllocated = 3,
  kInlineBool = 4,
};

// This constexpr function returns the storage kind for the given flag value
// type.
template <typename T>
static constexpr FlagValueStorageKind StorageKind() {
  return std::is_same_v<T, bool> ? FlagValueStorageKind::kInlineBool
         : FlagUseValueAndInitBitStorage<T>::value
             ? FlagValueStorageKind::kValueAndInitBit
         : FlagUseOneWordStorage<T>::value
             ? FlagValueStorageKind::kOneWordAtomic
         : FlagUseSequenceLockStorage<T>::value
             ? FlagValueStorageKind::kSequenceLocked
             : FlagValueStorageKind::kHeapAllocated;
}

// This is a base class for the storage classes used by kOneWordAtomic and
// kValueAndInitBit storage kinds. It literally just stores the one word value
// as an atomic. By default, it is initialized to a magic value that is unlikely
// a valid value for the flag value type.
struct FlagOneWordValue {
  constexpr static int64_t Uninitialized() {
    return static_cast<int64_t>(0xababababababababll);
  }

  constexpr FlagOneWordValue() : value(Uninitialized()) {}
  constexpr explicit FlagOneWordValue(int64_t v) : value(v) {}
  std::atomic<int64_t> value;
};

// This class represents a memory layout used by kValueAndInitBit storage kind.
template <typename T>
struct alignas(8) FlagValueAndInitBit {
  T value;
  // Use an int instead of a bool to guarantee that a non-zero value has
  // a bit set.
  uint8_t init;
};

// This class implements an aligned pointer with two options stored via masks
// in unused bits of the pointer value (due to alignment requirement).
//  - IsUnprotectedReadCandidate - indicates that the value can be switched to
//    unprotected read without a lock.
//  - HasBeenRead - indicates that the value has been read at least once.
//  - AllowsUnprotectedRead - combination of the two options above and indicates
//    that the value can now be read without a lock.
// Further details of these options and their use is covered in the description
// of the FlagValue<T, FlagValueStorageKind::kHeapAllocated> specialization.
class MaskedPointer {
 public:
  using mask_t = uintptr_t;
  using ptr_t = void*;

  static constexpr int RequiredAlignment() { return 4; }

  constexpr MaskedPointer() : ptr_(nullptr) {}
  constexpr explicit MaskedPointer(ptr_t rhs) : ptr_(rhs) {}
  MaskedPointer(ptr_t rhs, bool is_candidate);

  MaskedPointer(const MaskedPointer& rhs) = default;
  MaskedPointer& operator=(const MaskedPointer& rhs) = default;

  void* Ptr() const {
    return reinterpret_cast<void*>(reinterpret_cast<mask_t>(ptr_) &
                                   kPtrValueMask);
  }
  bool AllowsUnprotectedRead() const {
    return (reinterpret_cast<mask_t>(ptr_) & kAllowsUnprotectedRead) ==
           kAllowsUnprotectedRead;
  }
  bool IsUnprotectedReadCandidate() const;
  bool HasBeenRead() const;

  void Set(FlagOpFn op, const void* src, bool is_candidate);
  void MarkAsRead();

 private:
  // Masks
  // Indicates that the flag value either default or originated from command
  // line.
  static constexpr mask_t kUnprotectedReadCandidate = 0x1u;
  // Indicates that flag has been read.
  static constexpr mask_t kHasBeenRead = 0x2u;
  static constexpr mask_t kAllowsUnprotectedRead =
      kUnprotectedReadCandidate | kHasBeenRead;
  static constexpr mask_t kPtrValueMask = ~kAllowsUnprotectedRead;

  void ApplyMask(mask_t mask);
  bool CheckMask(mask_t mask) const;

  ptr_t ptr_;
};

// This class implements a type erased storage of the heap allocated flag value.
// It is used as a base class for the storage class for kHeapAllocated storage
// kind. The initial_buffer is expected to have an alignment of at least
// MaskedPointer::RequiredAlignment(), so that the bits used by the
// MaskedPointer to store masks are set to 0. This guarantees that value starts
// in an uninitialized state.
struct FlagMaskedPointerValue {
  constexpr explicit FlagMaskedPointerValue(MaskedPointer::ptr_t initial_buffer)
      : value(MaskedPointer(initial_buffer)) {}

  std::atomic<MaskedPointer> value;
};

// This is the forward declaration for the template that represents a storage
// for the flag values. This template is expected to be explicitly specialized
// for each storage kind and it does not have a generic default
// implementation.
template <typename T,
          FlagValueStorageKind Kind = flags_internal::StorageKind<T>()>
struct FlagValue;

// This specialization represents the storage of flag values types with the
// kValueAndInitBit storage kind. It is based on the FlagOneWordValue class
// and relies on memory layout in FlagValueAndInitBit<T> to indicate that the
// value has been initialized or not.
template <typename T>
struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue {
  constexpr FlagValue() : FlagOneWordValue(0) {}
  bool Get(T& dst) const {
    int64_t storage = value.load(std::memory_order_acquire);
    if (ABSL_PREDICT_FALSE(storage == 0)) {
      // This assert is to ensure that the initialization inside FlagImpl::Init
      // is able to set init member correctly.
      static_assert(offsetof(FlagValueAndInitBit<T>, init) == sizeof(T),
                    "Unexpected memory layout of FlagValueAndInitBit");
      return false;
    }
    dst = absl::bit_cast<FlagValueAndInitBit<T>>(storage).value;
    return true;
  }
};

// This specialization represents the storage of flag values types with the
// kOneWordAtomic storage kind. It is based on the FlagOneWordValue class
// and relies on the magic uninitialized state of default constructed instead of
// FlagOneWordValue to indicate that the value has been initialized or not.
template <typename T>
struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue {
  constexpr FlagValue() : FlagOneWordValue() {}
  bool Get(T& dst) const {
    int64_t one_word_val = value.load(std::memory_order_acquire);
    if (ABSL_PREDICT_FALSE(one_word_val == FlagOneWordValue::Uninitialized())) {
      return false;
    }
    std::memcpy(&dst, static_cast<const void*>(&one_word_val), sizeof(T));
    return true;
  }
};

// This specialization represents the storage of flag values types with the
// kSequenceLocked storage kind. This storage is used by trivially copyable
// types with size greater than 8 bytes. This storage relies on uninitialized
// state of the SequenceLock to indicate that the value has been initialized or
// not. This storage also provides lock-free read access to the underlying
// value once it is initialized.
template <typename T>
struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> {
  constexpr FlagValue() : FlagValue(std::make_index_sequence<kNumWords>{}) {}

  bool Get(T& dst) const {
    // FlagImpl::SeqLock() computes `&seq_lock` via `AtomicBufferValue() +
    // kNumWords`, so verify no padding or fields sit between `value_words`
    // and `seq_lock`.
    static_assert(
        offsetof(FlagValue, seq_lock) - offsetof(FlagValue, value_words) ==
            kNumWords * sizeof(std::atomic<uint64_t>),
        "Unexpected memory layout of sequence-locked FlagValue");
    return seq_lock.TryRead(&dst, value_words, sizeof(T));
  }

  static constexpr size_t kNumWords =
      flags_internal::AlignUp(sizeof(T), sizeof(uint64_t)) / sizeof(uint64_t);

  alignas(T) alignas(
      std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords];
  flags_internal::SequenceLock seq_lock;

 private:
  template <size_t... Is>
  constexpr explicit FlagValue(std::index_sequence<Is...>)
      : value_words{(static_cast<void>(Is), uint64_t{0})...} {}
};

// This specialization represents the storage of flag values types with the
// kHeapAllocated storage kind. This is a storage of last resort and is used
// if none of other storage kinds are applicable.
//
// Generally speaking the values with this storage kind can't be accessed
// atomically and thus can't be read without holding a lock. If we would ever
// want to avoid the lock, we'd need to leak the old value every time new flag
// value is being set (since we are in danger of having a race condition
// otherwise).
//
// Instead of doing that, this implementation attempts to cater to some common
// use cases by allowing at most 2 values to be leaked - default value and
// value set from the command line.
//
// This specialization provides an initial buffer for the first flag value. This
// is where the default value is going to be stored. We attempt to reuse this
// buffer if possible, including storing the value set from the command line
// there.
//
// As long as we only read this value, we can access it without a lock (in
// practice we still use the lock for the very first read to be able set
// "has been read" option on this flag).
//
// If flag is specified on the command line we store the parsed value either
// in the internal buffer (if the default value never been read) or we leak the
// default value and allocate the new storage for the parse value. This value is
// also a candidate for an unprotected read. If flag is set programmatically
// after the command line is parsed, the storage for this value is going to be
// leaked. Note that in both scenarios we are not going to have a real leak.
// Instead we'll store the leaked value pointers in the internal freelist to
// avoid triggering the memory leak checker complains.
//
// If the flag is ever set programmatically, it stops being the candidate for an
// unprotected read, and any follow up access to the flag value requires a lock.
// Note that if the value if set programmatically before the command line is
// parsed, we can switch back to enabling unprotected reads for that value.
template <typename T>
struct FlagValue<T, FlagValueStorageKind::kHeapAllocated>
    : FlagMaskedPointerValue {
  // We const initialize the value with unmasked pointer to the internal buffer,
  // making sure it is not a candidate for unprotected read. This way we can
  // ensure Init is done before any access to the flag value.
  constexpr FlagValue() : FlagMaskedPointerValue(&buffer[0]) {}

  bool Get(T& dst) const {
    MaskedPointer ptr_value = value.load(std::memory_order_acquire);

    if (ABSL_PREDICT_TRUE(ptr_value.AllowsUnprotectedRead())) {
      ::new (static_cast<void*>(&dst)) T(*static_cast<T*>(ptr_value.Ptr()));
      return true;
    }
    return false;
  }

  alignas(MaskedPointer::RequiredAlignment()) alignas(
      T) char buffer[sizeof(T)]{};
};

// Specialization for bool flags whose value is stored inline in FlagImpl.
template <typename T>
struct FlagValue<T, FlagValueStorageKind::kInlineBool> {};

///////////////////////////////////////////////////////////////////////////////
// Flag callback auxiliary structs.

// Signature for the mutation callback used by watched Flags
// The callback is noexcept.
// TODO(rogeeff): add noexcept after C++17 support is added.
using FlagCallbackFunc = void (*)();

struct FlagCallback {
  FlagCallbackFunc func;
  absl::Mutex guard;  // Guard for concurrent callback invocations.
};

///////////////////////////////////////////////////////////////////////////////
// Flag implementation, which does not depend on flag value type.
// The class encapsulates the Flag's data and access to it.

struct DynValueDeleter {
  explicit DynValueDeleter(FlagOpFn op_arg = nullptr);
  void operator()(void* ptr) const;

  FlagOpFn op;
};

inline constexpr uint8_t kNoBuiltinOp = 0xFF;

template <typename T>
struct BuiltinFlagOpIdImpl {
  static constexpr uint8_t kValue = kNoBuiltinOp;
};
template <>
struct BuiltinFlagOpIdImpl<bool> {
  static constexpr uint8_t kValue = 0;
};
template <>
struct BuiltinFlagOpIdImpl<short> {  // NOLINT
  static constexpr uint8_t kValue = 1;
};
template <>
struct BuiltinFlagOpIdImpl<unsigned short> {  // NOLINT
  static constexpr uint8_t kValue = 2;
};
template <>
struct BuiltinFlagOpIdImpl<int> {
  static constexpr uint8_t kValue = 3;
};
template <>
struct BuiltinFlagOpIdImpl<unsigned int> {
  static constexpr uint8_t kValue = 4;
};
template <>
struct BuiltinFlagOpIdImpl<long> {  // NOLINT
  static constexpr uint8_t kValue = 5;
};
template <>
struct BuiltinFlagOpIdImpl<unsigned long> {  // NOLINT
  static constexpr uint8_t kValue = 6;
};
template <>
struct BuiltinFlagOpIdImpl<long long> {  // NOLINT
  static constexpr uint8_t kValue = 7;
};
template <>
struct BuiltinFlagOpIdImpl<unsigned long long> {  // NOLINT
  static constexpr uint8_t kValue = 8;
};
template <>
struct BuiltinFlagOpIdImpl<float> {
  static constexpr uint8_t kValue = 9;
};
template <>
struct BuiltinFlagOpIdImpl<double> {
  static constexpr uint8_t kValue = 10;
};
template <>
struct BuiltinFlagOpIdImpl<std::string> {
  static constexpr uint8_t kValue = 11;
};
template <>
struct BuiltinFlagOpIdImpl<std::vector<std::string>> {
  static constexpr uint8_t kValue = 12;
};
template <>
struct BuiltinFlagOpIdImpl<absl::int128> {
  static constexpr uint8_t kValue = 13;
};
template <>
struct BuiltinFlagOpIdImpl<absl::uint128> {
  static constexpr uint8_t kValue = 14;
};
template <>
struct BuiltinFlagOpIdImpl<absl::LogSeverity> {
  static constexpr uint8_t kValue = 15;
};
template <>
struct BuiltinFlagOpIdImpl<absl::Duration> {
  static constexpr uint8_t kValue = 23;
};
template <>
struct BuiltinFlagOpIdImpl<absl::Time> {
  static constexpr uint8_t kValue = 24;
};
template <typename U>
struct BuiltinFlagOpIdImpl<std::optional<U>> {
  static constexpr uint8_t kValue = static_cast<uint8_t>(
      std::is_same_v<U, bool>             ? 16
      : std::is_same_v<U, int>            ? 17
      : std::is_same_v<U, long>           ? 18  // NOLINT
      : std::is_same_v<U, long long>      ? 19  // NOLINT
      : std::is_same_v<U, unsigned long>  ? 20  // NOLINT
      : std::is_same_v<U, double>         ? 21
      : std::is_same_v<U, std::string>    ? 22
      : std::is_same_v<U, absl::Duration> ? 25
                                          : kNoBuiltinOp);
};

template <typename T>
constexpr uint8_t BuiltinFlagOpId() {
  return BuiltinFlagOpIdImpl<T>::kValue;
}

struct FlagMetaHeader {
  constexpr static uint8_t PackKinds(FlagValueStorageKind vk, FlagHelpKind hk,
                                     FlagDefaultKind dk) {
    return static_cast<uint8_t>((static_cast<uint8_t>(vk) & 0x7u) |
                                ((static_cast<uint8_t>(hk) & 0x1u) << 3) |
                                ((static_cast<uint8_t>(dk) & 0x3u) << 4));
  }

  uint8_t kinds;
  uint8_t builtin_op_id;
};

// Pointer-based descriptor for built-in types (32 B, omits FlagOpFn pointer).
struct BuiltinPtrFlagMeta {
  constexpr BuiltinPtrFlagMeta(FlagMetaHeader hdr_arg, const char* name_arg,
                               FlagHelpMsg help_arg,
                               FlagDefaultSrc default_value_arg)
      : hdr(hdr_arg),
        name(name_arg),
        help(help_arg),
        default_value(default_value_arg) {}

  FlagMetaHeader hdr;
  const char* const name;
  const FlagHelpMsg help;
  const FlagDefaultSrc default_value;
};

// Full pointer-based descriptor for custom/enum types (40 B).
struct FlagMeta {
  constexpr FlagMeta(FlagMetaHeader hdr_arg, const char* name_arg,
                     FlagHelpMsg help_arg, FlagDefaultSrc default_value_arg,
                     FlagOpFn op_arg)
      : hdr(hdr_arg),
        name(name_arg),
        help(help_arg),
        default_value(default_value_arg),
        op(op_arg) {}

  FlagMetaHeader hdr;
  // Flag's name passed to ABSL_FLAG as second arg.
  const char* const name;
  // Help message literal or function to generate it.
  const FlagHelpMsg help;
  // Initial default value source specified in ABSL_FLAG.
  const FlagDefaultSrc default_value;
  // Type-specific operations vtable.
  const FlagOpFn op;
};

template <typename T>
constexpr FlagMeta MakeFlagMeta(const char* name, FlagHelpArg help,
                                FlagDefaultArg default_arg) {
  return FlagMeta(
      FlagMetaHeader{FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(),
                                               help.kind, default_arg.kind),
                     BuiltinFlagOpId<T>()},
      name, help.source, default_arg.source, &FlagOps<T>);
}

template <typename T>
constexpr BuiltinPtrFlagMeta MakeBuiltinPtrFlagMeta(
    const char* name, FlagHelpArg help, FlagDefaultArg default_arg) {
  return BuiltinPtrFlagMeta(
      FlagMetaHeader{FlagMetaHeader::PackKinds(flags_internal::StorageKind<T>(),
                                               help.kind, default_arg.kind),
                     BuiltinFlagOpId<T>()},
      name, help.source, default_arg.source);
}

template <typename ValueT, typename GenT, typename HelpGenT,
          typename HelpStorageT,
          bool IsBuiltin = (BuiltinFlagOpId<ValueT>() != kNoBuiltinOp)>
struct FlagMetaSelector {
  using type = BuiltinPtrFlagMeta;
  static constexpr BuiltinPtrFlagMeta Make(const char* name,
                                           const HelpStorageT& help_storage) {
    return MakeBuiltinPtrFlagMeta<ValueT>(name, HelpArg<HelpGenT>(help_storage),
                                          DefaultArg<ValueT, GenT>(0));
  }
};

template <typename ValueT, typename GenT, typename HelpGenT,
          typename HelpStorageT>
struct FlagMetaSelector<ValueT, GenT, HelpGenT, HelpStorageT, false> {
  using type = FlagMeta;
  static constexpr FlagMeta Make(const char* name,
                                 const HelpStorageT& help_storage) {
    return MakeFlagMeta<ValueT>(name, HelpArg<HelpGenT>(help_storage),
                                DefaultArg<ValueT, GenT>(0));
  }
};

template <typename ValueT, typename GenT, typename HelpGenT, size_t NameLen,
          typename HelpStorageT>
constexpr typename FlagMetaSelector<ValueT, GenT, HelpGenT, HelpStorageT>::type
SelectFlagMeta(const char* name, const HelpStorageT& help_storage, int) {
  return FlagMetaSelector<ValueT, GenT, HelpGenT, HelpStorageT>::Make(
      name, help_storage);
}

class FlagState;

// These are only used as constexpr global objects.
// They do not use a virtual destructor to simplify their implementation.
// They are not destroyed except at program exit, so leaks do not matter.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
#endif
class FlagImpl final : public CommandLineFlag {
 private:
  static constexpr uint8_t kMaxInlineModCount = 7;

  struct MuState {
    uint8_t def_kind : 2;
    bool modified : 1;
    bool on_command_line : 1;
    bool has_callback : 1;
    uint8_t mod_count : 3;

    constexpr FlagDefaultKind DefaultKind() const {
      return static_cast<FlagDefaultKind>(def_kind);
    }
  };
  static_assert(sizeof(MuState) == 1, "MuState must be 1 byte");
  static_assert(std::atomic<MuState>::is_always_lock_free,
                "std::atomic<MuState> must be lock-free");

 public:
  constexpr explicit FlagImpl(const FlagMetaHeader* meta)
      : meta_(meta),
        init_control_{},
        file_id_(0),
        mu_state_(MuState{static_cast<uint8_t>((meta->kinds >> 4) & 0x3u),
                          false, false, false, 0}),
        inline_bool_value_(0) {}

  constexpr FlagImpl(const FlagMeta* meta, FlagHelpKind /*help_kind*/,
                     FlagValueStorageKind /*value_kind*/,
                     FlagDefaultKind default_kind)
      : meta_(&meta->hdr),
        init_control_{},
        file_id_(0),
        mu_state_(MuState{static_cast<uint8_t>(default_kind), false, false,
                          false, 0}),
        inline_bool_value_(0) {}

  bool TryGetInlineBool(bool& dst) const {
    uint8_t s = inline_bool_value_.load(std::memory_order_acquire);
    if (ABSL_PREDICT_FALSE(s == 0)) return false;
    dst = static_cast<bool>(s & 1u);
    return true;
  }

  // Constant access methods
  int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(*DataGuard());
  bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(*DataGuard());
  void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard());
  void Read(bool* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) {
    *value = ReadOneBool();
  }
  template <typename T,
            std::enable_if_t<flags_internal::StorageKind<T>() ==
                                 FlagValueStorageKind::kOneWordAtomic,
                             int> = 0>
  void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) {
    int64_t v = ReadOneWord();
    std::memcpy(value, static_cast<const void*>(&v), sizeof(T));
  }
  template <typename T,
            std::enable_if_t<flags_internal::StorageKind<T>() ==
                                 FlagValueStorageKind::kValueAndInitBit,
                             int> = 0>
  void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) {
    *value = absl::bit_cast<FlagValueAndInitBit<T>>(ReadOneWord()).value;
  }

  // Mutating access methods
  void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard());
  void SetFilename(const char* filename);

  // Interfaces to operate on callbacks.
  void SetCallback(const FlagCallbackFunc mutation_callback)
      ABSL_LOCKS_EXCLUDED(*DataGuard());
  void InvokeCallback() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());

  // Used in read/write operations to validate source/target has correct type.
  // For example if flag is declared as absl::Flag<int> FLAGS_foo, a call to
  // absl::GetFlag(FLAGS_foo) validates that the type of FLAGS_foo is indeed
  // int. To do that we pass the assumed type id (which is deduced from type
  // int) as an argument `type_id`, which is in turn is validated against the
  // type id stored in flag object by flag definition statement.
  void AssertValidType(FlagFastTypeId type_id,
                       const std::type_info* (*gen_rtti)()) const;

 private:
  template <typename T>
  friend class Flag;
  friend class FlagState;

  FlagOpFn op() const;
  const BuiltinPtrFlagMeta* PtrMeta() const {
    return reinterpret_cast<const BuiltinPtrFlagMeta*>(meta_);
  }

  // Ensures that `data_guard_` is initialized and returns it.
  absl::Mutex* DataGuard() const
      ABSL_LOCK_RETURNED(reinterpret_cast<absl::Mutex*>(data_guard_));
  // Returns heap allocated value of type T initialized with default value.
  std::unique_ptr<void, DynValueDeleter> MakeInitValue() const
      ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());
  // Flag initialization called via absl::call_once.
  void Init();

  // Offset value access methods. One per storage kind. These methods to not
  // respect const correctness, so be very careful using them.

  // This is a shared helper routine which encapsulates most of the magic. Since
  // it is only used inside the routines below, which are defined in
  // flag.cc, we can define it in that file as well.
  template <typename StorageT>
  StorageT* OffsetValue() const;

  // The same as above, but used for sequencelock-protected storage.
  std::atomic<uint64_t>* AtomicBufferValue() const;

  // Accessor for the SequenceLock inside FlagValue<T, kSequenceLocked>.
  flags_internal::SequenceLock& SeqLock() const;

  // This is an accessor for a value stored as one word atomic. Returns a
  // mutable reference to an atomic value.
  std::atomic<int64_t>& OneWordValue() const;

  std::atomic<MaskedPointer>& PtrStorage() const;

  // Attempts to parse supplied `value` string. If parsing is successful,
  // returns new value. Otherwise returns nullptr.
  std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value,
                                                  std::string& err) const
      ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());
  // Stores the flag value based on the pointer to the source.
  void StoreValue(const void* src, ValueSource source)
      ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());

  // Copy the flag data, protected by `SeqLock()` into `dst`.
  //
  // REQUIRES: ValueStorageKind() == kSequenceLocked.
  void ReadSequenceLockedData(void* dst) const
      ABSL_LOCKS_EXCLUDED(*DataGuard());

  FlagHelpKind HelpSourceKind() const {
    return static_cast<FlagHelpKind>((meta_->kinds >> 3) & 0x1u);
  }
  FlagValueStorageKind ValueStorageKind() const {
    return static_cast<FlagValueStorageKind>(meta_->kinds & 0x7u);
  }

  // CommandLineFlag interface implementation
  absl::string_view Name() const override;
  absl::string_view TypeName() const override;
  std::string Filename() const override;
  std::string Help() const override;
  FlagFastTypeId TypeId() const override;
  bool IsSpecifiedOnCommandLine() const override
      ABSL_LOCKS_EXCLUDED(*DataGuard());
  std::string DefaultValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard());
  std::string CurrentValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard());
  bool ValidateInputValue(absl::string_view value) const override
      ABSL_LOCKS_EXCLUDED(*DataGuard());
  void CheckDefaultValueParsingRoundtrip() const override
      ABSL_LOCKS_EXCLUDED(*DataGuard());

  int64_t ModificationCount() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());
  // Lock-free check used by SaveState() and RestoreState() to skip lazy
  // initialization on flags that have never been mutated. Safe without
  // DataGuard() because all mutations update mu_state_ under DataGuard()
  // with release semantics after writing the flag value, and observing a
  // zero modification count means the flag still holds its initial
  // compile-time state.
  bool IsUnmodifiedFast() const {
    return mu_state_.load(std::memory_order_acquire).mod_count == 0;
  }
  bool CanSkipInitOnSave() const;
  void RecordMutation() ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());

  // Interfaces to save and restore flags to/from persistent state.
  // Returns current flag state or nullptr if flag does not support
  // saving and restoring a state.
  std::unique_ptr<FlagStateInterface> SaveState() override
      ABSL_LOCKS_EXCLUDED(*DataGuard());

  // Restores the flag state to the supplied state object. If there is
  // nothing to restore returns false. Otherwise returns true.
  bool RestoreState(const FlagState& flag_state)
      ABSL_LOCKS_EXCLUDED(*DataGuard());

  bool ParseFrom(absl::string_view value, FlagSettingMode set_mode,
                 ValueSource source, std::string& error) override
      ABSL_LOCKS_EXCLUDED(*DataGuard());

  // Pointer to immutable flag descriptor in .rodata / .data.rel.ro.
  const FlagMetaHeader* const meta_;

  // Per-flag mutex guard, placement-initialized in Init().
  alignas(sizeof(void*)) mutable char data_guard_[sizeof(absl::Mutex)]{};

  // Unique tag for absl::call_once call to initialize this flag.
  absl::once_flag init_control_;

  // Deduplicated filename index assigned on registration (0 = unset).
  uint16_t file_id_;

  // Atomic state bitfields (def_kind, modified, on_command_line, has_callback,
  // and saturating 3-bit mod_count where 7 indicates overflow in side table).
  // All writes occur under DataGuard() with release semantics; SaveState() and
  // RestoreState() perform lock-free acquire loads to fast-path unmodified
  // flags without triggering lazy initialization.
  std::atomic<MuState> mu_state_;

  // Inline atomic storage for Flag<bool> (0 = uninit, 2 = false, 3 = true).
  mutable std::atomic<uint8_t> inline_bool_value_;
};
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif

///////////////////////////////////////////////////////////////////////////////
// The Flag object parameterized by the flag's value type. This class implements
// flag reflection handle interface.

template <typename T,
          FlagValueStorageKind Kind = flags_internal::StorageKind<T>()>
struct FlagStorage {
  constexpr explicit FlagStorage(const FlagMetaHeader* meta)
      : impl_(meta), value_() {}
  constexpr FlagStorage(const FlagMeta* meta, FlagHelpKind help_kind,
                        FlagDefaultKind default_kind)
      : impl_(meta, help_kind, Kind, default_kind), value_() {}

  // The implementation depends on value_ field to be placed exactly after the
  // impl_ field, so that impl_ can figure out the offset to the value and
  // access it.
  FlagImpl impl_;
  FlagValue<T, Kind> value_;
};

template <typename T>
struct FlagStorage<T, FlagValueStorageKind::kInlineBool> {
  constexpr explicit FlagStorage(const FlagMetaHeader* meta) : impl_(meta) {}
  constexpr FlagStorage(const FlagMeta* meta, FlagHelpKind help_kind,
                        FlagDefaultKind default_kind)
      : impl_(meta, help_kind, FlagValueStorageKind::kInlineBool,
              default_kind) {}

  FlagImpl impl_;
};

template <typename T>
class Flag : private FlagStorage<T> {
 public:
  constexpr explicit Flag(const FlagMetaHeader* meta) : FlagStorage<T>(meta) {}

  constexpr Flag(const FlagMeta* meta, FlagHelpKind help_kind,
                 FlagDefaultKind default_kind)
      : FlagStorage<T>(meta, help_kind, default_kind) {}

  // CommandLineFlag interface
  absl::string_view Name() const { return this->impl_.Name(); }
  std::string Filename() const { return this->impl_.Filename(); }
  std::string Help() const { return this->impl_.Help(); }
  // Do not use. To be removed.
  bool IsSpecifiedOnCommandLine() const {
    return this->impl_.IsSpecifiedOnCommandLine();
  }
  std::string DefaultValue() const { return this->impl_.DefaultValue(); }
  std::string CurrentValue() const { return this->impl_.CurrentValue(); }

 private:
  template <typename, bool>
  friend class FlagRegistrar;
  friend class FlagImplPeer;

  T Get() const {
    // See implementation notes in CommandLineFlag::Get().
    union U {
      T value;
      U() {}
      ~U() { value.~T(); }
    };
    U u;

#if !defined(NDEBUG)
    this->impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>);
#endif

    if constexpr (flags_internal::StorageKind<T>() ==
                  FlagValueStorageKind::kInlineBool) {
      if (ABSL_PREDICT_FALSE(!this->impl_.TryGetInlineBool(u.value))) {
        this->impl_.Read(&u.value);
      }
    } else {
      if (ABSL_PREDICT_FALSE(!this->value_.Get(u.value))) {
        this->impl_.Read(&u.value);
      }
    }
    return std::move(u.value);
  }
  void Set(const T& v) {
    this->impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>);
    this->impl_.Write(&v);
  }

  // Access to the reflection.
  const CommandLineFlag& Reflect() const { return this->impl_; }
};

///////////////////////////////////////////////////////////////////////////////
// Trampoline for friend access

class FlagImplPeer {
 public:
  template <typename T, typename FlagType>
  static T InvokeGet(const FlagType& flag) {
    return flag.Get();
  }
  template <typename FlagType, typename T>
  static void InvokeSet(FlagType& flag, const T& v) {
    flag.Set(v);
  }
  template <typename FlagType>
  static const CommandLineFlag& InvokeReflect(const FlagType& f) {
    return f.Reflect();
  }
};

///////////////////////////////////////////////////////////////////////////////
// Implementation of Flag value specific operations routine.
template <typename T>
void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) {
  struct AlignedSpace {
    alignas(MaskedPointer::RequiredAlignment()) alignas(
        T) unsigned char buf[sizeof(T)];
  };
  using Allocator = std::allocator<AlignedSpace>;
  switch (op) {
    case FlagOp::kAlloc: {
      Allocator alloc;
      return std::allocator_traits<Allocator>::allocate(alloc, 1);
    }
    case FlagOp::kDelete: {
      T* p = static_cast<T*>(v2);
      p->~T();
      Allocator alloc;
      std::allocator_traits<Allocator>::deallocate(
          alloc, reinterpret_cast<AlignedSpace*>(p), 1);
      return nullptr;
    }
    case FlagOp::kCopy:
      *static_cast<T*>(v2) = *static_cast<const T*>(v1);
      return nullptr;
    case FlagOp::kCopyConstruct:
      if constexpr (std::is_default_constructible_v<T>) {
        if (v1 == nullptr) {
          new (v2) T();
          return nullptr;
        }
      }
      new (v2) T(*static_cast<const T*>(v1));
      return nullptr;
    case FlagOp::kSizeof:
      return reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(T)));
    case FlagOp::kFastTypeId:
      return absl::bit_cast<void*>(absl::FastTypeId<T>());
    case FlagOp::kRuntimeTypeId:
      return const_cast<std::type_info*>(GenRuntimeTypeId<T>());
    case FlagOp::kParse: {
      // Initialize the temporary instance of type T based on current value in
      // destination (which is going to be flag's default value).
      T temp(*static_cast<T*>(v2));
      if (!absl::ParseFlag<T>(*static_cast<const absl::string_view*>(v1), &temp,
                              static_cast<std::string*>(v3))) {
        return nullptr;
      }
      *static_cast<T*>(v2) = std::move(temp);
      return v2;
    }
    case FlagOp::kUnparse:
      *static_cast<std::string*>(v2) =
          absl::UnparseFlag<T>(*static_cast<const T*>(v1));
      return nullptr;
    case FlagOp::kValueOffset: {
      // Round sizeof(FlagImp) to a multiple of alignof(FlagValue<T>) to get the
      // offset of the data.
      size_t round_to = alignof(FlagValue<T>);
      size_t offset = (sizeof(FlagImpl) + round_to - 1) / round_to * round_to;
      return reinterpret_cast<void*>(offset);
    }
    case FlagOp::kTypeName:
      *static_cast<absl::string_view*>(v2) = GenTypeName<T>();
      return nullptr;
  }
  return nullptr;
}

///////////////////////////////////////////////////////////////////////////////
// This class facilitates Flag object registration and tail expression-based
// flag definition, for example:
// ABSL_FLAG(int, foo, 42, "Foo help").OnUpdate(NotifyFooWatcher);
struct FlagRegistrarEmpty {};
template <typename T, bool do_register>
class FlagRegistrar {
 public:
  constexpr explicit FlagRegistrar(Flag<T>& flag, const char* filename)
      : flag_(flag) {
    if (filename != nullptr) {
      flag_.impl_.SetFilename(filename);
    }
    if (do_register) {
      flags_internal::RegisterCommandLineFlag(flag_.impl_, filename);
    }
  }

  FlagRegistrar OnUpdate(FlagCallbackFunc cb) && {
    flag_.impl_.SetCallback(cb);
    return *this;
  }

  // Makes the registrar die gracefully as an empty struct on a line where
  // registration happens. Registrar objects are intended to live only as
  // temporary.
  constexpr operator FlagRegistrarEmpty() const { return {}; }  // NOLINT

 private:
  Flag<T>& flag_;  // Flag being registered (not owned).
};

///////////////////////////////////////////////////////////////////////////////
// Test only API
uint64_t NumLeakedFlagValues();

}  // namespace flags_internal
ABSL_NAMESPACE_END
}  // namespace absl

#endif  // ABSL_FLAGS_INTERNAL_FLAG_H_