#include "universal_container/adaptive_sequence.hpp"

#include <algorithm>
#include <array>
#include <chrono>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <filesystem>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <iterator>
#include <limits>
#include <list>
#include <map>
#include <numeric>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
#include <tuple>
#include <type_traits>
#include <utility>
#include <vector>

#ifndef UC_BUILD_PROFILE
#define UC_BUILD_PROFILE "unknown"
#endif

namespace {

using Clock = std::chrono::steady_clock;
using Nanoseconds = std::chrono::nanoseconds;
volatile std::uint64_t benchmark_sink = 0;

template <std::size_t Bytes>
struct Blob {
    static_assert(Bytes >= sizeof(std::uint64_t));
    std::uint64_t key = 0;
    std::array<std::byte, Bytes - sizeof(std::uint64_t)> payload{};
};

struct NonTrivial {
    std::uint64_t key = 0;
    std::string payload;

    NonTrivial() = default;
    explicit NonTrivial(std::uint64_t value)
        : key(value), payload(40, static_cast<char>('a' + value % 26)) {}
    NonTrivial(const NonTrivial&) = default;
    NonTrivial(NonTrivial&&) noexcept = default;
    NonTrivial& operator=(const NonTrivial&) = default;
    NonTrivial& operator=(NonTrivial&&) noexcept = default;
    ~NonTrivial() { key ^= 0; }
};

template <class T>
T make_value(std::uint64_t key) {
    if constexpr (std::is_integral_v<T>) {
        return static_cast<T>(key);
    } else if constexpr (requires(T value) { value.key; value.payload; }
                         && !std::is_same_v<T, NonTrivial>) {
        T value{};
        value.key = key;
        for (std::size_t i = 0; i < value.payload.size(); ++i) {
            value.payload[i] = static_cast<std::byte>((key + i * 17U) & 0xffU);
        }
        return value;
    } else {
        return T(key);
    }
}

template <class T>
std::uint64_t value_key(const T& value) noexcept {
    if constexpr (std::is_integral_v<T>) {
        return static_cast<std::uint64_t>(value);
    } else {
        return value.key;
    }
}

inline void digest_value(std::uint64_t& digest,
                         std::uint64_t key,
                         std::size_t index) noexcept {
    // FNV-style rolling digest with the logical position folded in.  This is
    // a correctness guard, not a cryptographic hash.
    digest ^= key + 0x9e3779b97f4a7c15ULL
           + static_cast<std::uint64_t>(index) * 0xbf58476d1ce4e5b9ULL;
    digest *= 1099511628211ULL;
}

inline std::uint64_t finish_digest(std::uint64_t digest,
                                   std::size_t size) noexcept {
    digest_value(digest, static_cast<std::uint64_t>(size), size);
    return digest;
}

inline std::size_t initial_reserve_capacity(std::size_t size) {
    const auto slack = std::max<std::size_t>(1024, size / 8);
    if (size > std::numeric_limits<std::size_t>::max() - slack) {
        throw std::length_error("initial reserve capacity overflow");
    }
    return size + slack;
}

template <class T>
std::string type_name();
template <> std::string type_name<std::uint32_t>() { return "uint32"; }
template <> std::string type_name<std::uint64_t>() { return "uint64"; }
template <> std::string type_name<Blob<16>>() { return "blob16"; }
template <> std::string type_name<Blob<32>>() { return "blob32"; }
template <> std::string type_name<Blob<64>>() { return "blob64"; }
template <> std::string type_name<NonTrivial>() { return "nontrivial_string"; }

enum class TraceKind : std::uint8_t { read, insert, erase, append, maintenance };

struct TraceStep {
    TraceKind kind = TraceKind::read;
    std::uint64_t selector = 0;
    std::uint64_t value = 0;
    bool localized = false;
};

struct Workload {
    std::string name;
    double read_fraction = 1.0;
    bool sequential = false;
    bool append_only = false;
    bool localized = false;
    bool bursty = false;
    bool shape_shift = false;
    std::size_t uniform_edit_operations = 0;
    std::size_t localized_edit_operations = 0;
    std::size_t read_operations = 0;

    [[nodiscard]] std::size_t explicit_operation_count() const noexcept {
        return uniform_edit_operations + localized_edit_operations + read_operations;
    }
};

struct Options {
    std::string suite = "all";
    std::string scale = "smoke";
    std::filesystem::path output = "results/benchmarks/manual/results.csv";
    std::filesystem::path summary = "results/benchmarks/manual/summary.csv";
    std::uint64_t seed = 0x8d12'74ab'3c55'9011ULL;
    std::size_t leaf = 512;
    std::size_t fanout = 64;
    std::size_t levels = 4;
    std::size_t repeats_override = 0;
    // Targeted large-scale cell.  Separate processes run these cells in
    // parallel so a single invocation stays memory-bounded and resumable.
    std::size_t n_override = 0;
    std::size_t operations_override = 0;
    std::size_t edit_ppm = 0;
    std::string large_pattern = "steady_uniform";
};

struct Result {
    std::string suite;
    std::string type;
    std::string structure;
    std::string workload;
    std::size_t n = 0;
    std::size_t operations = 0;
    std::size_t repeat = 0;
    std::uint64_t seed = 0;
    uc::TieredConfig config{};
    uc::TieredConfig final_config{};
    double construction_ns = 0.0;
    std::size_t initial_bytes = 0;
    double total_ns = 0.0;
    double ns_per_operation = 0.0;
    double p50_ns = 0.0;
    double p95_ns = 0.0;
    double p99_ns = 0.0;
    double max_ns = 0.0;
    double max_maintenance_ns = 0.0;
    double max_transition_ns = 0.0;
    std::uint64_t checksum = 0;
    std::size_t allocated_bytes = 0;
    std::uint64_t switches = 0;
    std::uint64_t shape_rebuilds = 0;
    std::uint64_t leaf_rebuilds = 0;
    std::uint64_t directory_rebuilds = 0;
    std::uint64_t policy_evaluations = 0;
    std::size_t last_forecast_operations = 0;
    std::size_t last_evidence_windows = 0;
    double last_vector_cost = 0.0;
    double last_tiered_cost = 0.0;
    double last_current_cost = 0.0;
    double last_conversion_cost = 0.0;
    double last_expected_saving = 0.0;
    double last_local_edit_fraction = 0.0;
    double last_edit_fraction = 0.0;
    std::string final_mode;
    std::string status = "ok";
};

std::string csv_escape(std::string_view value) {
    std::string result;
    result.reserve(value.size() + 2);
    result.push_back('"');
    for (const char character : value) {
        if (character == '"') {
            result.push_back('"');
        }
        result.push_back(character);
    }
    result.push_back('"');
    return result;
}

void write_results(const std::filesystem::path& path, const std::vector<Result>& results) {
    if (!path.parent_path().empty()) {
        std::filesystem::create_directories(path.parent_path());
    }
    std::ofstream output(path, std::ios::binary);
    if (!output) {
        throw std::runtime_error("cannot open benchmark output: " + path.string());
    }
    output << "profile,suite,type,structure,workload,n,operations,repeat,seed,"
              "leaf,fanout,levels,final_leaf,final_fanout,final_levels,"
              "construction_ns,initial_bytes,total_ns,ns_per_op,p50_ns,p95_ns,p99_ns,max_ns,"
              "max_maintenance_ns,max_transition_ns,checksum,allocated_bytes,"
              "switches,shape_rebuilds,leaf_rebuilds,directory_rebuilds,"
              "policy_evaluations,last_forecast_operations,last_evidence_windows,last_vector_cost,"
              "last_tiered_cost,last_current_cost,last_conversion_cost,"
              "last_expected_saving,last_local_edit_fraction,last_edit_fraction,"
              "final_mode,status\n";
    output << std::setprecision(17);
    for (const auto& result : results) {
        output << csv_escape(UC_BUILD_PROFILE) << ','
               << csv_escape(result.suite) << ','
               << csv_escape(result.type) << ','
               << csv_escape(result.structure) << ','
               << csv_escape(result.workload) << ','
               << result.n << ',' << result.operations << ',' << result.repeat << ','
               << result.seed << ',' << result.config.leaf_capacity << ','
               << result.config.directory_fanout << ',' << result.config.directory_levels << ','
               << result.final_config.leaf_capacity << ','
               << result.final_config.directory_fanout << ','
               << result.final_config.directory_levels << ','
               << result.construction_ns << ',' << result.initial_bytes << ','
               << result.total_ns << ',' << result.ns_per_operation << ','
               << result.p50_ns << ',' << result.p95_ns << ',' << result.p99_ns << ','
               << result.max_ns << ',' << result.max_maintenance_ns << ','
               << result.max_transition_ns << ','
               << result.checksum << ',' << result.allocated_bytes << ','
               << result.switches << ',' << result.shape_rebuilds << ','
               << result.leaf_rebuilds << ',' << result.directory_rebuilds << ','
               << result.policy_evaluations << ',' << result.last_forecast_operations << ','
               << result.last_evidence_windows << ','
               << result.last_vector_cost << ',' << result.last_tiered_cost << ','
               << result.last_current_cost << ',' << result.last_conversion_cost << ','
               << result.last_expected_saving << ','
               << result.last_local_edit_fraction << ','
               << result.last_edit_fraction << ','
               << csv_escape(result.final_mode) << ','
               << csv_escape(result.status) << '\n';
    }
}

double percentile(std::vector<double> values, double quantile) {
    if (values.empty()) {
        return 0.0;
    }
    std::sort(values.begin(), values.end());
    const auto index = static_cast<std::size_t>(
        std::ceil(quantile * static_cast<double>(values.size())) - 1.0);
    return values[std::min(index, values.size() - 1)];
}

std::vector<TraceStep> make_trace(const Workload& workload,
                                  std::size_t operation_count,
                                  std::uint64_t seed) {
    std::mt19937_64 random(seed);
    std::vector<TraceStep> trace;
    trace.reserve(operation_count + 3);
    bool next_edit_is_insert = true;

    if (workload.explicit_operation_count() != 0) {
        auto append_edits = [&](std::size_t count, bool localized) {
            for (std::size_t i = 0; i < count; ++i) {
                const auto kind = next_edit_is_insert ? TraceKind::insert : TraceKind::erase;
                next_edit_is_insert = !next_edit_is_insert;
                trace.push_back({kind, random(), random(), localized});
            }
        };
        auto append_maintenance = [&] {
            trace.push_back({TraceKind::maintenance, random(), random(), false});
        };

        if (workload.uniform_edit_operations != 0) {
            append_edits(workload.uniform_edit_operations, false);
            append_maintenance();
        }
        if (workload.localized_edit_operations != 0) {
            append_edits(workload.localized_edit_operations, true);
            append_maintenance();
        }
        for (std::size_t i = 0; i < workload.read_operations; ++i) {
            trace.push_back({TraceKind::read, random(), random(), false});
        }
        if (workload.read_operations != 0) {
            append_maintenance();
        }
        return trace;
    }

    for (std::size_t i = 0; i < operation_count; ++i) {
        TraceKind kind = TraceKind::read;
        bool localized = workload.localized;
        if (workload.append_only) {
            kind = TraceKind::append;
        } else if (workload.shape_shift) {
            const auto uniform_end = operation_count * 40 / 100;
            const auto localized_end = operation_count * 80 / 100;
            if (i == uniform_end || i == localized_end || i + 1 == operation_count) {
                kind = TraceKind::maintenance;
            } else if (i < localized_end) {
                kind = next_edit_is_insert ? TraceKind::insert : TraceKind::erase;
                next_edit_is_insert = !next_edit_is_insert;
                localized = i >= uniform_end;
            }
        } else if (workload.bursty) {
            const auto first_read_end = operation_count * 45 / 100;
            const auto edit_end = operation_count * 55 / 100;
            if (i == edit_end || i + 1 == operation_count) {
                kind = TraceKind::maintenance;
            } else if (i >= first_read_end && i < edit_end) {
                kind = next_edit_is_insert ? TraceKind::insert : TraceKind::erase;
                next_edit_is_insert = !next_edit_is_insert;
            }
        } else {
            const auto draw = static_cast<double>(random() % 1'000'000) / 1'000'000.0;
            if (draw >= workload.read_fraction) {
                kind = next_edit_is_insert ? TraceKind::insert : TraceKind::erase;
                next_edit_is_insert = !next_edit_is_insert;
            }
        }
        trace.push_back({kind, random(), random(), localized});
    }
    return trace;
}

std::string large_workload_name(std::string_view pattern, std::size_t edit_ppm) {
    return std::string(pattern) + "_edit_ppm_" + std::to_string(edit_ppm);
}

std::vector<TraceStep> make_large_trace(std::size_t operations,
                                        std::size_t edit_ppm,
                                        std::string_view pattern,
                                        std::uint64_t seed) {
    if (operations == 0 || edit_ppm > 1'000'000) {
        throw std::invalid_argument("invalid large trace operation count or edit ppm");
    }

    const auto scaled = static_cast<std::uint64_t>(operations)
                      * static_cast<std::uint64_t>(edit_ppm);
    auto edit_count = static_cast<std::size_t>(
        (scaled + 500'000ULL) / 1'000'000ULL);
    if (edit_ppm != 0) {
        edit_count = std::max<std::size_t>(1, edit_count);
    }
    edit_count = std::min(edit_count, operations);

    std::mt19937_64 random(seed);
    std::vector<TraceStep> trace;
    trace.reserve(operations + 3);
    bool next_edit_is_insert = true;
    auto append_edit = [&](bool localized) {
        const auto kind = next_edit_is_insert ? TraceKind::insert : TraceKind::erase;
        next_edit_is_insert = !next_edit_is_insert;
        trace.push_back({kind, random(), random(), localized});
    };
    auto append_read = [&] {
        trace.push_back({TraceKind::read, random(), random(), false});
    };
    auto append_maintenance = [&] {
        trace.push_back({TraceKind::maintenance, random(), random(), false});
    };

    if (pattern == "phase") {
        const auto uniform_edits = edit_count / 2;
        const auto localized_edits = edit_count - uniform_edits;
        for (std::size_t i = 0; i < uniform_edits; ++i) append_edit(false);
        if (uniform_edits != 0) append_maintenance();
        for (std::size_t i = 0; i < localized_edits; ++i) append_edit(true);
        if (localized_edits != 0) append_maintenance();
        for (std::size_t i = edit_count; i < operations; ++i) append_read();
        append_maintenance();
        return trace;
    }

    const bool localized = pattern == "steady_localized"
                        || pattern == "bursty_localized";
    if (pattern == "bursty_uniform" || pattern == "bursty_localized") {
        const auto edit_begin = (operations - edit_count) / 2;
        const auto edit_end = edit_begin + edit_count;
        for (std::size_t i = 0; i < operations; ++i) {
            if (i >= edit_begin && i < edit_end) append_edit(localized);
            else append_read();
            if (i + 1 == edit_end && edit_count != 0) append_maintenance();
        }
        append_maintenance();
        return trace;
    }

    if (pattern != "steady_uniform" && pattern != "steady_localized") {
        throw std::invalid_argument("unknown large trace pattern");
    }

    // Sample without replacement: the count is exact, while the seed jitters
    // edit slots so policy windows cannot alias with a periodic schedule.
    auto remaining_edits = edit_count;
    for (std::size_t i = 0; i < operations; ++i) {
        const auto remaining_slots = operations - i;
        const bool choose_edit = remaining_edits == remaining_slots
                              || (remaining_edits != 0
                                  && random() % remaining_slots < remaining_edits);
        if (choose_edit) {
            append_edit(localized);
            --remaining_edits;
        } else {
            append_read();
        }
    }
    append_maintenance();
    return trace;
}

std::size_t trace_position(std::uint64_t selector,
                           std::size_t size,
                           bool localized,
                           bool allow_end) noexcept {
    const auto range = size + (allow_end ? 1U : 0U);
    if (range == 0) {
        return 0;
    }
    if (!localized || size < 100) {
        return static_cast<std::size_t>(selector % range);
    }
    const auto window = std::max<std::size_t>(1, size / 100);
    const auto middle = size / 2;
    const auto offset = static_cast<std::size_t>(selector % (2 * window + 1));
    const auto begin = middle > window ? middle - window : 0;
    return std::min(begin + offset, range - 1);
}

template <class T>
std::vector<T> make_initial(std::size_t n, std::uint64_t seed) {
    std::vector<T> values;
    values.reserve(n);
    std::uint64_t state = seed;
    for (std::size_t i = 0; i < n; ++i) {
        state = state * 6364136223846793005ULL + 1442695040888963407ULL;
        values.push_back(make_value<T>(state));
    }
    return values;
}

template <class T>
class VectorAdapter {
public:
    VectorAdapter(const std::vector<T>& initial, uc::TieredConfig)
    {
        values_.reserve(initial_reserve_capacity(initial.size()));
        values_.insert(values_.end(), initial.begin(), initial.end());
    }
    const T& get(std::size_t index) const { return values_[index]; }
    void insert(std::size_t index, T value) {
        values_.insert(values_.begin() + static_cast<std::ptrdiff_t>(index), std::move(value));
    }
    void erase(std::size_t index) {
        values_.erase(values_.begin() + static_cast<std::ptrdiff_t>(index));
    }
    void append(T value) { values_.push_back(std::move(value)); }
    std::uint64_t scan() const {
        std::uint64_t sum = 0;
        for (const auto& value : values_) sum += value_key(value);
        return sum;
    }
    std::uint64_t digest() const noexcept {
        std::uint64_t result = 1469598103934665603ULL;
        std::size_t index = 0;
        for (const auto& value : values_) {
            digest_value(result, value_key(value), index++);
        }
        return finish_digest(result, values_.size());
    }
    std::size_t size() const noexcept { return values_.size(); }
    std::size_t bytes() const noexcept { return values_.capacity() * sizeof(T); }
    std::uint64_t switches() const noexcept { return 0; }
    std::string mode() const { return "vector"; }
    static std::string name() { return "std::vector_reserved"; }
private:
    std::vector<T> values_;
};

template <class T>
class DequeAdapter {
public:
    DequeAdapter(const std::vector<T>& initial, uc::TieredConfig)
        : values_(initial.begin(), initial.end()) {}
    const T& get(std::size_t index) const { return values_[index]; }
    void insert(std::size_t index, T value) {
        values_.insert(values_.begin() + static_cast<std::ptrdiff_t>(index), std::move(value));
    }
    void erase(std::size_t index) {
        values_.erase(values_.begin() + static_cast<std::ptrdiff_t>(index));
    }
    void append(T value) { values_.push_back(std::move(value)); }
    std::uint64_t scan() const {
        std::uint64_t sum = 0;
        for (const auto& value : values_) sum += value_key(value);
        return sum;
    }
    std::uint64_t digest() const noexcept {
        std::uint64_t result = 1469598103934665603ULL;
        std::size_t index = 0;
        for (const auto& value : values_) {
            digest_value(result, value_key(value), index++);
        }
        return finish_digest(result, values_.size());
    }
    std::size_t size() const noexcept { return values_.size(); }
    std::size_t bytes() const noexcept {
        return values_.size() * sizeof(T) + (values_.size() / 64 + 2) * sizeof(void*);
    }
    std::uint64_t switches() const noexcept { return 0; }
    std::string mode() const { return "deque"; }
    static std::string name() { return "std::deque"; }
private:
    std::deque<T> values_;
};

template <class T>
class ListAdapter {
public:
    ListAdapter(const std::vector<T>& initial, uc::TieredConfig)
        : values_(initial.begin(), initial.end()) {}
    const T& get(std::size_t index) const { return *at(index); }
    void insert(std::size_t index, T value) { values_.insert(at(index), std::move(value)); }
    void erase(std::size_t index) { values_.erase(at(index)); }
    void append(T value) { values_.push_back(std::move(value)); }
    std::uint64_t scan() const {
        std::uint64_t sum = 0;
        for (const auto& value : values_) sum += value_key(value);
        return sum;
    }
    std::uint64_t digest() const noexcept {
        std::uint64_t result = 1469598103934665603ULL;
        std::size_t index = 0;
        for (const auto& value : values_) {
            digest_value(result, value_key(value), index++);
        }
        return finish_digest(result, values_.size());
    }
    std::size_t size() const noexcept { return values_.size(); }
    std::size_t bytes() const noexcept {
        return values_.size() * (sizeof(T) + 2 * sizeof(void*));
    }
    std::uint64_t switches() const noexcept { return 0; }
    std::string mode() const { return "list"; }
    static std::string name() { return "std::list_index_semantics"; }
private:
    typename std::list<T>::iterator at(std::size_t index) {
        if (index < values_.size() / 2) {
            auto iterator = values_.begin();
            std::advance(iterator, static_cast<std::ptrdiff_t>(index));
            return iterator;
        }
        auto iterator = values_.end();
        std::advance(iterator, -static_cast<std::ptrdiff_t>(values_.size() - index));
        return iterator;
    }
    typename std::list<T>::const_iterator at(std::size_t index) const {
        if (index < values_.size() / 2) {
            auto iterator = values_.begin();
            std::advance(iterator, static_cast<std::ptrdiff_t>(index));
            return iterator;
        }
        auto iterator = values_.end();
        std::advance(iterator, -static_cast<std::ptrdiff_t>(values_.size() - index));
        return iterator;
    }
    std::list<T> values_;
};

enum class AdaptivePolicyVariant : std::uint8_t {
    compare_default,
    aggressive,
    moderate,
    balanced
};

template <class T, bool ForcedTiered, bool EagerReads = false,
          AdaptivePolicyVariant PolicyVariant = AdaptivePolicyVariant::compare_default>
class AdaptiveAdapter {
public:
    AdaptiveAdapter(const std::vector<T>& initial, uc::TieredConfig config)
        : values_(config, make_policy(config)) {
        values_.reserve(initial_reserve_capacity(initial.size()));
        for (const auto& value : initial) values_.push_back(value);
        if constexpr (ForcedTiered) {
            values_.force_tiered_mode();
        } else {
            values_.policy().reset();
            values_.enable_auto_mode();
            if constexpr (EagerReads) {
                values_.set_read_adaptation_mode(uc::ReadAdaptationMode::eager_nonconst);
            }
        }
    }
    const T& get(std::size_t index) { return values_[index]; }
    void insert(std::size_t index, T value) { values_.insert(index, std::move(value)); }
    void erase(std::size_t index) { values_.erase(index); }
    void append(T value) { values_.push_back(std::move(value)); }
    std::uint64_t scan() const {
        std::uint64_t sum = 0;
        values_.for_each([&](const T& value) { sum += value_key(value); });
        return sum;
    }
    std::uint64_t digest() const noexcept {
        std::uint64_t result = 1469598103934665603ULL;
        std::size_t index = 0;
        values_.for_each([&](const T& value) {
            digest_value(result, value_key(value), index++);
        });
        return finish_digest(result, values_.size());
    }
    std::size_t size() const noexcept { return values_.size(); }
    std::size_t bytes() const noexcept { return values_.allocated_bytes(); }
    std::uint64_t switches() const noexcept {
        if constexpr (ForcedTiered) {
            return 0;
        } else {
            const auto& telemetry = values_.policy().telemetry();
            return telemetry.vector_to_tiered + telemetry.tiered_to_vector;
        }
    }
    std::uint64_t shape_rebuilds() const noexcept {
        if constexpr (ForcedTiered) {
            return 0;
        } else {
            return values_.policy().telemetry().tiered_rebuilds;
        }
    }
    std::uint64_t leaf_rebuilds() const noexcept {
        if constexpr (ForcedTiered) {
            return 0;
        } else {
            return values_.policy().telemetry().tiered_leaf_rebuilds;
        }
    }
    std::uint64_t directory_rebuilds() const noexcept {
        if constexpr (ForcedTiered) {
            return 0;
        } else {
            return values_.policy().telemetry().tiered_directory_rebuilds;
        }
    }
    uc::AdaptationTelemetry telemetry() const noexcept {
        if constexpr (ForcedTiered) {
            return {};
        } else {
            return values_.policy().telemetry();
        }
    }
    uc::TieredConfig config() const noexcept { return values_.tiered_config(); }
    bool maintenance() { return values_.adapt_now(); }
    std::string mode() const {
        return values_.mode() == uc::StorageMode::vector ? "vector" : "tiered";
    }
    static std::string name() {
        if constexpr (ForcedTiered) return "tiered_forced";
        if constexpr (PolicyVariant == AdaptivePolicyVariant::aggressive) {
            return EagerReads ? "adaptive_shape_aggressive_eager"
                              : "adaptive_shape_aggressive_deferred";
        }
        if constexpr (PolicyVariant == AdaptivePolicyVariant::moderate) {
            return EagerReads ? "adaptive_shape_moderate_eager"
                              : "adaptive_shape_moderate_deferred";
        }
        if constexpr (PolicyVariant == AdaptivePolicyVariant::balanced) {
            return EagerReads ? "adaptive_shape_balanced_eager"
                              : "adaptive_shape_balanced_deferred";
        }
        if constexpr (EagerReads) return "adaptive_shape_eager";
        return "adaptive_shape_deferred";
    }
    std::string structure_name() const {
        if constexpr (ForcedTiered) {
            return "tiered_forced_leaf_" +
                   std::to_string(values_.tiered_config().leaf_capacity);
        }
        return name();
    }
private:
    static uc::CostModelPolicy make_policy(uc::TieredConfig config) {
        uc::AdaptationConfig adaptation;
        adaptation.minimum_tiered_size = 8 * 1024;
        adaptation.evaluation_interval = 256;
        adaptation.forecast_operations = 128 * 1024;
        if constexpr (PolicyVariant == AdaptivePolicyVariant::balanced) {
            adaptation.edit_evaluation_interval = 64;
            adaptation.minimum_edit_observations = 256;
            adaptation.minimum_observations = 4096;
            adaptation.minimum_residency_operations = 1024;
        } else if constexpr (PolicyVariant == AdaptivePolicyVariant::moderate
                             || PolicyVariant == AdaptivePolicyVariant::compare_default) {
            adaptation.edit_evaluation_interval = 32;
            adaptation.minimum_edit_observations = 128;
            adaptation.minimum_observations = 2048;
            adaptation.minimum_residency_operations = 256;
        } else {
            // Aggressive control: useful on long edit phases, deliberately
            // expected to overreact to the 128-edit negative control.
            adaptation.edit_evaluation_interval = 32;
            adaptation.minimum_edit_observations = 64;
            adaptation.minimum_observations = 1024;
            adaptation.minimum_residency_operations = 64;
        }
        return uc::CostModelPolicy(adaptation, config);
    }
    uc::AdaptiveSequence<T, false, uc::CostModelPolicy> values_;
};

template <class Adapter, class T>
Result execute_trace(const std::vector<T>& initial,
                     const std::vector<TraceStep>& trace,
                     const Workload& workload,
                     uc::TieredConfig config,
                     std::string suite,
                     std::size_t repeat,
                     std::uint64_t seed) {
    const auto construction_started = Clock::now();
    Adapter container(initial, config);
    const auto construction_finished = Clock::now();
    std::vector<double> edit_latencies;
    edit_latencies.reserve(trace.size() / 2);
    std::uint64_t checksum = 0;
    std::size_t semantic_operations = 0;
    double max_maintenance_ns = 0.0;
    double max_transition_ns = 0.0;
    auto transition_count = [&]() noexcept {
        auto count = container.switches();
        if constexpr (requires(const Adapter& value) { value.shape_rebuilds(); }) {
            count += container.shape_rebuilds();
        }
        return count;
    };

    const auto started = Clock::now();
    if (workload.sequential) {
        const auto scans = std::max<std::size_t>(1, trace.size());
        semantic_operations = scans * container.size();
        for (std::size_t i = 0; i < scans; ++i) {
            checksum ^= container.scan() + i;
        }
    } else {
        for (const auto& step : trace) {
            if (step.kind == TraceKind::maintenance) {
                if constexpr (requires(Adapter& value) { value.maintenance(); }) {
                    const auto transitions_before = transition_count();
                    const auto maintenance_started = Clock::now();
                    const bool changed = container.maintenance();
                    const auto maintenance_finished = Clock::now();
                    const auto transitions_after = transition_count();
                    if (changed || transitions_after != transitions_before) {
                        const auto latency = static_cast<double>(
                            std::chrono::duration_cast<Nanoseconds>(
                                maintenance_finished - maintenance_started).count());
                        edit_latencies.push_back(latency);
                        max_maintenance_ns = std::max(max_maintenance_ns, latency);
                        max_transition_ns = std::max(max_transition_ns, latency);
                    }
                }
                continue;
            }
            if (step.kind == TraceKind::read) {
                const auto position = trace_position(step.selector, container.size(),
                                                     workload.localized || step.localized, false);
                checksum += value_key(container.get(position));
                ++semantic_operations;
                continue;
            }

            const auto transitions_before = transition_count();
            const auto operation_started = Clock::now();
            if (step.kind == TraceKind::insert) {
                const auto position = trace_position(step.selector, container.size(),
                                                     workload.localized || step.localized, true);
                container.insert(position, make_value<T>(step.value));
            } else if (step.kind == TraceKind::erase) {
                if (container.size() > 1) {
                    const auto position = trace_position(step.selector, container.size(),
                                                         workload.localized || step.localized, false);
                    container.erase(position);
                }
            } else {
                container.append(make_value<T>(step.value));
            }
            const auto operation_finished = Clock::now();
            const auto latency = static_cast<double>(
                std::chrono::duration_cast<Nanoseconds>(
                    operation_finished - operation_started).count());
            edit_latencies.push_back(latency);
            if (transition_count() != transitions_before) {
                max_transition_ns = std::max(max_transition_ns, latency);
            }
            ++semantic_operations;
        }
    }
    const auto finished = Clock::now();
    checksum ^= container.digest();
    benchmark_sink = checksum;

    Result result;
    result.suite = std::move(suite);
    result.type = type_name<T>();
    if constexpr (requires(const Adapter& value) { value.structure_name(); }) {
        result.structure = container.structure_name();
    } else {
        result.structure = Adapter::name();
    }
    result.workload = workload.name;
    result.n = initial.size();
    result.operations = semantic_operations;
    result.repeat = repeat;
    result.seed = seed;
    result.config = config;
    if constexpr (requires(const Adapter& value) { value.config(); }) {
        result.final_config = container.config();
    } else {
        result.final_config = {0, 0, 0};
    }
    result.construction_ns = static_cast<double>(
        std::chrono::duration_cast<Nanoseconds>(
            construction_finished - construction_started).count());
    result.initial_bytes = initial.capacity() * sizeof(T);
    result.total_ns = static_cast<double>(
        std::chrono::duration_cast<Nanoseconds>(finished - started).count());
    result.ns_per_operation = semantic_operations == 0
        ? 0.0 : result.total_ns / static_cast<double>(semantic_operations);
    result.p50_ns = percentile(edit_latencies, 0.50);
    result.p95_ns = percentile(edit_latencies, 0.95);
    result.p99_ns = percentile(edit_latencies, 0.99);
    result.max_ns = edit_latencies.empty()
        ? 0.0 : *std::max_element(edit_latencies.begin(), edit_latencies.end());
    result.max_maintenance_ns = max_maintenance_ns;
    result.max_transition_ns = max_transition_ns;
    result.checksum = checksum;
    result.allocated_bytes = container.bytes();
    result.switches = container.switches();
    if constexpr (requires(const Adapter& value) { value.shape_rebuilds(); }) {
        result.shape_rebuilds = container.shape_rebuilds();
        result.leaf_rebuilds = container.leaf_rebuilds();
        result.directory_rebuilds = container.directory_rebuilds();
    }
    if constexpr (requires(const Adapter& value) { value.telemetry(); }) {
        const auto telemetry = container.telemetry();
        result.policy_evaluations = telemetry.evaluations;
        result.last_forecast_operations = telemetry.last_forecast_operations;
        result.last_evidence_windows = telemetry.last_evidence_windows;
        result.last_vector_cost = telemetry.last_vector_cost_per_operation;
        result.last_tiered_cost = telemetry.last_tiered_cost_per_operation;
        result.last_current_cost = telemetry.last_current_cost_per_operation;
        result.last_conversion_cost = telemetry.last_conversion_cost;
        result.last_expected_saving = telemetry.last_expected_saving;
        result.last_local_edit_fraction = telemetry.last_local_edit_fraction;
        result.last_edit_fraction = telemetry.last_edit_fraction;
    }
    result.final_mode = container.mode();
    return result;
}

std::size_t repeats_for(const Options& options) {
    if (options.repeats_override != 0) return options.repeats_override;
    if (options.scale == "full") return 9;
    if (options.scale == "quick") return 5;
    return 2;
}

std::size_t operation_count_for(const Options& options,
                                const Workload& workload,
                                std::size_t n) {
    if (workload.explicit_operation_count() != 0) {
        return workload.explicit_operation_count();
    }
    if (workload.sequential) {
        if (options.scale == "full") return std::max<std::size_t>(3, 100'000'000 / n);
        if (options.scale == "quick") return std::max<std::size_t>(2, 20'000'000 / n);
        return std::max<std::size_t>(1, 1'000'000 / n);
    }
    if (options.scale == "full") return workload.read_fraction > 0.99 ? 2'000'000 : 100'000;
    if (options.scale == "quick") return workload.read_fraction > 0.99 ? 500'000 : 30'000;
    return workload.read_fraction > 0.99 ? 50'000 : 5'000;
}

template <class T>
std::size_t operation_count_for_type(const Options& options,
                                     const Workload& workload,
                                     std::size_t n,
                                     bool preserve_semantic_operations) {
    auto operations = operation_count_for(options, workload, n);
    if (preserve_semantic_operations || workload.explicit_operation_count() != 0) {
        return operations;
    }
    if (!workload.sequential && workload.read_fraction < 0.999
        && sizeof(T) > sizeof(std::uint64_t)) {
        const auto width_divisor = std::max<std::size_t>(
            2, sizeof(T) / sizeof(std::uint64_t));
        operations = std::max<std::size_t>(1'000, operations / width_divisor);
    }
    if constexpr (std::is_same_v<T, NonTrivial>) {
        if (!workload.sequential && workload.read_fraction < 0.999) {
            operations = std::max<std::size_t>(1'000, operations / 4);
        }
    }
    return operations;
}

long double estimated_edit_operations(const Workload& workload,
                                      std::size_t operations) noexcept {
    if (workload.explicit_operation_count() != 0) {
        return static_cast<long double>(workload.uniform_edit_operations
                                      + workload.localized_edit_operations);
    }
    if (workload.sequential || workload.append_only) return 0.0L;
    if (workload.shape_shift) return static_cast<long double>(operations) * 0.80L;
    if (workload.bursty) return static_cast<long double>(operations) * 0.10L;
    return static_cast<long double>(operations)
         * std::clamp(1.0L - static_cast<long double>(workload.read_fraction),
                      0.0L, 1.0L);
}

template <class T>
long double estimated_middle_shift_bytes(const Workload& workload,
                                         std::size_t operations,
                                         std::size_t n) noexcept {
    auto estimate = estimated_edit_operations(workload, operations)
                  * static_cast<long double>(n)
                  * static_cast<long double>(sizeof(T)) * 0.5L;
    if constexpr (!std::is_trivially_copyable_v<T>) {
        estimate *= 2.0L;
    }
    return estimate;
}

long double comparison_deque_edit_budget(const Options& options) noexcept {
    constexpr long double gib = 1024.0L * 1024.0L * 1024.0L;
    if (options.scale == "full") return 32.0L * gib;
    if (options.scale == "quick") return 16.0L * gib;
    return 8.0L * gib;
}

std::vector<std::size_t> tuning_sizes(const Options& options) {
    if (options.scale == "full") return {10'000, 100'000, 1'000'000, 10'000'000};
    if (options.scale == "quick") return {10'000, 100'000, 1'000'000};
    return {10'000, 100'000};
}

std::vector<std::size_t> comparison_sizes(const Options& options) {
    if (options.scale == "full") return {1'000, 10'000, 100'000, 1'000'000, 10'000'000};
    if (options.scale == "quick") return {1'000, 10'000, 100'000, 1'000'000};
    return {1'000, 10'000, 100'000};
}

template <class T, class Adapter>
void append_adapter_results(std::vector<Result>& results,
                            const Options& options,
                            const std::vector<T>& initial,
                            const Workload& workload,
                            uc::TieredConfig config,
                            std::string suite,
                            std::size_t n,
                            bool skip = false,
                            bool preserve_semantic_operations = false) {
    const auto repeats = repeats_for(options);
    const auto operations = operation_count_for_type<T>(
        options, workload, n, preserve_semantic_operations);
    for (std::size_t repeat = 0; repeat < repeats; ++repeat) {
        const auto seed = options.seed ^ (static_cast<std::uint64_t>(n) << 17U)
                        ^ (static_cast<std::uint64_t>(repeat) * 0x9e3779b97f4a7c15ULL)
                        ^ std::hash<std::string>{}(workload.name);
        if (skip) {
            Result result;
            result.suite = suite;
            result.type = type_name<T>();
            result.structure = Adapter::name();
            result.workload = workload.name;
            result.n = n;
            result.operations = operations;
            result.repeat = repeat;
            result.seed = seed;
            result.config = config;
            if constexpr (requires(const Adapter& value) { value.config(); }) {
                result.final_config = config;
            } else {
                result.final_config = {0, 0, 0};
            }
            result.status = "skipped_cost_guard";
            results.push_back(std::move(result));
            continue;
        }
        const auto trace = make_trace(workload, operations, seed);
        results.push_back(execute_trace<Adapter>(initial, trace, workload, config,
                                                 suite, repeat, seed));
    }
}

std::vector<Result> run_tuning(const Options& options) {
    std::vector<Result> results;
    const std::vector<Workload> workloads{
        {"random_access", 1.0, false, false, false, false},
        {"sequential", 1.0, true, false, false, false},
        {"uniform_edit", 0.0, false, false, false, false},
        {"localized_edit", 0.0, false, false, true, false},
        {"mixed_80_read", 0.80, false, false, false, false}
    };
    const std::array<std::size_t, 5> leaves{64, 128, 256, 512, 1024};
    const std::array<std::size_t, 3> levels{2, 3, 4};

    for (const auto leaf : leaves) {
        for (const auto level : levels) {
            const uc::TieredConfig config{leaf, options.fanout, level};
            for (const auto n : tuning_sizes(options)) {
                const auto initial = make_initial<std::uint32_t>(n, options.seed ^ n);
                for (const auto& workload : workloads) {
                    append_adapter_results<std::uint32_t,
                        AdaptiveAdapter<std::uint32_t, true>>(
                            results, options, initial, workload, config, "tune", n);
                }
            }
            std::cout << "tuned candidate leaf=" << leaf << " levels=" << level << '\n';
        }
    }
    return results;
}

template <class T>
void run_type_comparison(std::vector<Result>& results,
                         const Options& options,
                         const std::vector<std::size_t>& sizes,
                         const std::vector<Workload>& workloads,
                         uc::TieredConfig config,
                         std::string_view suite_name = "compare") {
    for (const auto n : sizes) {
        // Avoid paging when a large payload plus several transient copies would
        // exceed a conservative per-case budget.
        if (n > (2ULL * 1024 * 1024 * 1024) / std::max<std::size_t>(1, sizeof(T))) {
            continue;
        }
        const auto initial = make_initial<T>(n, options.seed ^ n ^ sizeof(T));
        for (const auto& workload : workloads) {
            const auto operations = operation_count_for_type<T>(
                options, workload, n, false);
            const auto estimated_shift = estimated_middle_shift_bytes<T>(
                workload, operations, n);
            const auto deque_edit_budget = comparison_deque_edit_budget(options);
            const bool skip_deque = estimated_shift > deque_edit_budget;
            const bool skip_list = (n > 10'000 && !workload.sequential)
                                || estimated_shift > deque_edit_budget;

            append_adapter_results<T, VectorAdapter<T>>(
                results, options, initial, workload, config, std::string(suite_name), n);
            append_adapter_results<T, DequeAdapter<T>>(
                results, options, initial, workload, config, std::string(suite_name), n,
                skip_deque);
            append_adapter_results<T, ListAdapter<T>>(
                results, options, initial, workload, config, std::string(suite_name), n,
                skip_list);
            for (const auto leaf : std::array<std::size_t, 5>{64, 128, 256, 512, 1024}) {
                auto fixed_config = config;
                fixed_config.leaf_capacity = leaf;
                append_adapter_results<T, AdaptiveAdapter<T, true>>(
                    results, options, initial, workload, fixed_config,
                    std::string(suite_name), n);
            }
            append_adapter_results<T, AdaptiveAdapter<T, false>>(
                results, options, initial, workload, config, std::string(suite_name), n);
            append_adapter_results<T, AdaptiveAdapter<T, false, true>>(
                results, options, initial, workload, config, std::string(suite_name), n);
        }
    }
}

std::vector<Result> run_comparison(const Options& options) {
    std::vector<Result> results;
    const uc::TieredConfig config{options.leaf, options.fanout, options.levels};
    const std::vector<Workload> workloads{
        {"random_access", 1.0, false, false, false, false},
        {"sequential", 1.0, true, false, false, false},
        {"append_only", 0.0, false, true, false, false},
        {"read_99_99", 0.9999, false, false, false, false},
        {"read_99", 0.99, false, false, false, false},
        {"mixed_80_read", 0.80, false, false, false, false},
        {"edit_50_read", 0.50, false, false, false, false},
        {"uniform_edit", 0.0, false, false, false, false},
        {"localized_edit", 0.0, false, false, true, false},
        {"bursty_edit", 1.0, false, false, false, true},
        // Uniform edits favour fewer/larger leaves, then a hot edit window
        // favours smaller leaves, followed by reads and an explicit safe
        // maintenance point.  Conversion/rebuild time remains in the trace.
        {"phase_uniform_local_read", 0.20, false, false, false, false, true}
    };
    const auto sizes = comparison_sizes(options);
    run_type_comparison<std::uint32_t>(results, options, sizes, workloads, config);

    // Smoke still validates multiple object sizes but uses one representative N.
    const std::vector<std::size_t> secondary_sizes = options.scale == "full"
        ? std::vector<std::size_t>{10'000, 100'000, 1'000'000}
        : options.scale == "quick"
            ? std::vector<std::size_t>{10'000, 100'000}
            : std::vector<std::size_t>{10'000};
    const std::vector<Workload> secondary_workloads{
        workloads[0], workloads[1], workloads[5], workloads[7], workloads[8]
    };
    run_type_comparison<std::uint64_t>(results, options, secondary_sizes,
                                       secondary_workloads, config);
    run_type_comparison<Blob<16>>(results, options, secondary_sizes,
                                  secondary_workloads, config);
    run_type_comparison<Blob<32>>(results, options, secondary_sizes,
                                  secondary_workloads, config);
    run_type_comparison<Blob<64>>(results, options, secondary_sizes,
                                  secondary_workloads, config);
    run_type_comparison<NonTrivial>(results, options, secondary_sizes,
                                    secondary_workloads, config);
    return results;
}

std::vector<Result> run_large_cell(const Options& options) {
    if (options.n_override == 0 || options.operations_override == 0) {
        throw std::invalid_argument(
            "large suite requires positive --n and --operations");
    }

    std::vector<Result> results;
    const uc::TieredConfig initial_config{
        options.leaf, options.fanout, options.levels};
    const auto workload_name = large_workload_name(
        options.large_pattern, options.edit_ppm);
    Workload workload{workload_name,
        1.0 - static_cast<double>(options.edit_ppm) / 1'000'000.0};
    const auto initial = make_initial<std::uint32_t>(
        options.n_override, options.seed ^ options.n_override);
    const auto repeats = repeats_for(options);
    constexpr std::size_t candidate_count = 7;
    results.reserve(candidate_count * repeats);

    for (std::size_t repeat = 0; repeat < repeats; ++repeat) {
        const auto seed = options.seed
                        ^ (static_cast<std::uint64_t>(options.n_override) << 17U)
                        ^ (static_cast<std::uint64_t>(repeat)
                           * 0x9e3779b97f4a7c15ULL)
                        ^ std::hash<std::string>{}(workload_name);
        const auto trace = make_large_trace(
            options.operations_override, options.edit_ppm,
            options.large_pattern, seed);

        std::vector<std::function<void()>> runners;
        runners.reserve(candidate_count);
        runners.emplace_back([&] {
            results.push_back(execute_trace<VectorAdapter<std::uint32_t>>(
                initial, trace, workload, initial_config, "large", repeat, seed));
        });
        for (const auto leaf : std::array<std::size_t, 5>{64, 128, 256, 512, 1024}) {
            auto fixed_config = initial_config;
            fixed_config.leaf_capacity = leaf;
            runners.emplace_back([&, fixed_config] {
                results.push_back(execute_trace<
                    AdaptiveAdapter<std::uint32_t, true>>(
                    initial, trace, workload, fixed_config,
                    "large", repeat, seed));
            });
        }
        runners.emplace_back([&] {
            results.push_back(execute_trace<
                AdaptiveAdapter<std::uint32_t, false>>(
                initial, trace, workload, initial_config,
                "large", repeat, seed));
        });

        // One repeat still rotates candidate order by cell; additional repeats
        // continue the cycle.  This avoids always measuring adaptive last.
        const auto base_rotation = (
            options.n_override / 1'000
            + options.edit_ppm
            + std::hash<std::string>{}(options.large_pattern)) % runners.size();
        const auto rotation = (base_rotation + repeat) % runners.size();
        for (std::size_t i = 0; i < runners.size(); ++i) {
            runners[(rotation + i) % runners.size()]();
        }
    }
    return results;
}

std::vector<Workload> adaptation_workloads(const Options& options) {
    std::size_t phase_edits = 512;
    std::size_t phase_reads = 4'096;
    std::size_t short_edits = 32;
    std::size_t short_reads = 4'096;
    if (options.scale == "quick") {
        phase_edits = 4'096;
        phase_reads = 32'768;
        short_edits = 128;
        short_reads = 32'768;
    } else if (options.scale == "full") {
        phase_edits = 16'384;
        phase_reads = 131'072;
        // Keep the negative control below the balanced policy's 256-edit
        // evidence threshold while still exercising more work than smoke.
        short_edits = 192;
        short_reads = 131'072;
    }

    Workload phase{"phase_10_uniform_10_local_80_read", 0.80};
    phase.uniform_edit_operations = phase_edits;
    phase.localized_edit_operations = phase_edits;
    phase.read_operations = phase_reads;

    Workload short_burst{"short_burst_negative_control", 1.0};
    short_burst.uniform_edit_operations = short_edits;
    short_burst.read_operations = short_reads;
    return {std::move(phase), std::move(short_burst)};
}

long double adaptation_edit_budget(const Options& options) noexcept {
    constexpr long double gib = 1024.0L * 1024.0L * 1024.0L;
    if (options.scale == "full") return 256.0L * gib;
    if (options.scale == "quick") return 64.0L * gib;
    return 16.0L * gib;
}

template <class T>
void append_paired_adaptation_results(std::vector<Result>& results,
                                      const Options& options,
                                      const std::vector<T>& initial,
                                      const Workload& workload,
                                      uc::TieredConfig config,
                                      std::size_t n) {
    const auto repeats = repeats_for(options);
    for (std::size_t repeat = 0; repeat < repeats; ++repeat) {
        const auto seed = options.seed ^ (static_cast<std::uint64_t>(n) << 17U)
                        ^ (static_cast<std::uint64_t>(repeat) * 0x9e3779b97f4a7c15ULL)
                        ^ std::hash<std::string>{}(workload.name);
        const auto trace = make_trace(workload, workload.explicit_operation_count(), seed);

        std::vector<std::function<void()>> runners;
        runners.reserve(10);
        auto add_runner = [&](auto adapter_tag, uc::TieredConfig runner_config) {
            using Adapter = typename decltype(adapter_tag)::type;
            runners.emplace_back([&, runner_config] {
                results.push_back(execute_trace<Adapter>(
                    initial, trace, workload, runner_config, "adapt", repeat, seed));
            });
        };

        add_runner(std::type_identity<VectorAdapter<T>>{}, config);
        for (const auto leaf : std::array<std::size_t, 5>{64, 128, 256, 512, 1024}) {
            auto fixed_config = config;
            fixed_config.leaf_capacity = leaf;
            add_runner(std::type_identity<AdaptiveAdapter<T, true>>{}, fixed_config);
        }
        add_runner(std::type_identity<
            AdaptiveAdapter<T, false, false, AdaptivePolicyVariant::aggressive>>{}, config);
        add_runner(std::type_identity<
            AdaptiveAdapter<T, false, false, AdaptivePolicyVariant::moderate>>{}, config);
        add_runner(std::type_identity<
            AdaptiveAdapter<T, false, false, AdaptivePolicyVariant::balanced>>{}, config);
        add_runner(std::type_identity<
            AdaptiveAdapter<T, false, true, AdaptivePolicyVariant::balanced>>{}, config);

        // Every candidate consumes the exact same materialized trace.  A cyclic
        // order distributes ordinal positions across repeats.  Full
        // randomization remains a documented future refinement.
        const auto base_rotation = (n / 1'000
            + std::hash<std::string>{}(workload.name)) % runners.size();
        const auto rotation = (base_rotation + repeat) % runners.size();
        for (std::size_t i = 0; i < runners.size(); ++i) {
            runners[(rotation + i) % runners.size()]();
        }
    }
}

template <class T>
void run_adaptation_type(std::vector<Result>& results,
                         const Options& options,
                         const std::vector<std::size_t>& sizes,
                         const std::vector<Workload>& workloads,
                         uc::TieredConfig config) {
    for (const auto n : sizes) {
        if (n > (2ULL * 1024 * 1024 * 1024) / std::max<std::size_t>(1, sizeof(T))) {
            continue;
        }
        const auto initial = make_initial<T>(n, options.seed ^ n ^ sizeof(T));
        for (const auto& workload : workloads) {
            const auto operations = operation_count_for_type<T>(
                options, workload, n, true);
            if (estimated_middle_shift_bytes<T>(workload, operations, n)
                > adaptation_edit_budget(options)) {
                std::cout << "skipped adapt cost guard: type=" << type_name<T>()
                          << " workload=" << workload.name << " n=" << n << '\n';
                continue;
            }

            append_paired_adaptation_results(
                results, options, initial, workload, config, n);
        }
    }
}

std::vector<Result> run_adaptation_benchmark(const Options& options) {
    std::vector<Result> results;
    const uc::TieredConfig config{options.leaf, options.fanout, options.levels};
    const auto workloads = adaptation_workloads(options);
    const auto sizes = options.scale == "full"
        ? std::vector<std::size_t>{10'000, 100'000, 1'000'000, 10'000'000}
        : options.scale == "quick"
            ? std::vector<std::size_t>{10'000, 100'000, 1'000'000}
            : std::vector<std::size_t>{10'000, 100'000};
    run_adaptation_type<std::uint32_t>(results, options, sizes, workloads, config);

    const std::vector<std::size_t> secondary_sizes = options.scale == "full"
        ? std::vector<std::size_t>{100'000, 1'000'000}
        : std::vector<std::size_t>{100'000};
    run_adaptation_type<std::uint64_t>(
        results, options, secondary_sizes, workloads, config);
    run_adaptation_type<Blob<16>>(
        results, options, secondary_sizes, workloads, config);
    run_adaptation_type<Blob<32>>(
        results, options, secondary_sizes, workloads, config);
    run_adaptation_type<Blob<64>>(
        results, options, secondary_sizes, workloads, config);
    run_adaptation_type<NonTrivial>(
        results, options, secondary_sizes, workloads, config);
    return results;
}

std::vector<Result> run_index_benchmark(const Options& options) {
    std::vector<Result> results;
    const uc::TieredConfig config{options.leaf, options.fanout, options.levels};
    const auto repeats = repeats_for(options);
    const auto sizes = options.scale == "full"
        ? std::vector<std::size_t>{10'000, 100'000, 1'000'000, 10'000'000}
        : options.scale == "quick"
            ? std::vector<std::size_t>{10'000, 100'000, 1'000'000}
            : std::vector<std::size_t>{10'000, 100'000};
    const std::array<std::size_t, 3> cardinalities{
        16, 1024, std::numeric_limits<std::size_t>::max()
    };

    for (const auto n : sizes) {
        for (const auto requested_cardinality : cardinalities) {
            const auto cardinality = std::min(n, requested_cardinality);
            std::vector<std::uint32_t> initial;
            initial.reserve(n);
            for (std::size_t i = 0; i < n; ++i) {
                initial.push_back(static_cast<std::uint32_t>(i % cardinality));
            }
            const auto base_operations = options.scale == "full" ? 2'000'000U
                                       : options.scale == "quick" ? 500'000U : 50'000U;
            // Linear lookup is deliberately part of the baseline, so cap the
            // number of examined elements rather than accidentally running an
            // unbounded O(N * operations) job at N=10^7.
            const auto operations = std::min<std::size_t>(base_operations,
                std::max<std::size_t>(32, 50'000'000 / n));

            for (const std::string workload : {"contains", "find_all", "find_all_ids"}) {
                for (std::size_t repeat = 0; repeat < repeats; ++repeat) {
                    const auto seed = options.seed ^ n ^ (cardinality << 21U)
                                    ^ (repeat * 0x9e3779b97f4a7c15ULL);
                    std::mt19937_64 random(seed);
                    std::vector<std::uint32_t> queries;
                    queries.reserve(operations);
                    for (std::size_t i = 0; i < operations; ++i) {
                        queries.push_back((i & 1U) == 0
                            ? static_cast<std::uint32_t>(random() % cardinality)
                            : static_cast<std::uint32_t>(cardinality + 1 + random() % cardinality));
                    }

                    auto run_vector = [&]() {
                        std::uint64_t checksum = 0;
                        const auto started = Clock::now();
                        if (workload == "contains") {
                            for (const auto query : queries) {
                                checksum += std::find(initial.begin(), initial.end(), query)
                                         != initial.end();
                            }
                        } else {
                            for (const auto query : queries) {
                                std::vector<std::uint64_t> matches;
                                for (std::size_t index = 0; index < initial.size(); ++index) {
                                    if (initial[index] == query) matches.push_back(index);
                                }
                                checksum += matches.size();
                            }
                        }
                        const auto finished = Clock::now();
                        benchmark_sink = checksum;
                        Result result;
                        result.suite = "index"; result.type = "uint32";
                        result.structure = "std::vector_linear_lookup";
                        result.workload = workload + "_cardinality_" + std::to_string(cardinality);
                        result.n = n; result.operations = operations; result.repeat = repeat;
                        result.seed = seed; result.config = config;
                        result.final_config = {0, 0, 0}; result.checksum = checksum;
                        result.total_ns = static_cast<double>(
                            std::chrono::duration_cast<Nanoseconds>(finished - started).count());
                        result.ns_per_operation = result.total_ns / operations;
                        result.allocated_bytes = initial.capacity() * sizeof(std::uint32_t);
                        result.final_mode = "vector";
                        return result;
                    };

                    auto run_indexed = [&]() {
                        uc::AdaptiveSequence<std::uint32_t, true> indexed(config);
                        indexed.reserve(n);
                        for (const auto value : initial) indexed.push_back(value);
                        indexed.enable_auto_mode();
                        std::uint64_t checksum = 0;
                        const auto started = Clock::now();
                        if (workload == "contains") {
                            for (const auto query : queries) checksum += indexed.contains(query);
                        } else if (workload == "find_all") {
                            for (const auto query : queries) checksum += indexed.find_all(query).size();
                        } else {
                            for (const auto query : queries) checksum += indexed.find_all_ids(query).size();
                        }
                        const auto finished = Clock::now();
                        benchmark_sink = checksum;
                        Result result;
                        result.suite = "index"; result.type = "uint32";
                        result.structure = "adaptive_flat_hash_index";
                        result.workload = workload + "_cardinality_" + std::to_string(cardinality);
                        result.n = n; result.operations = operations; result.repeat = repeat;
                        result.seed = seed; result.config = config;
                        result.final_config = indexed.tiered_config(); result.checksum = checksum;
                        result.total_ns = static_cast<double>(
                            std::chrono::duration_cast<Nanoseconds>(finished - started).count());
                        result.ns_per_operation = result.total_ns / operations;
                        result.allocated_bytes = indexed.allocated_bytes();
                        result.final_mode = indexed.mode() == uc::StorageMode::vector
                            ? "vector" : "tiered";
                        return result;
                    };

                    results.push_back(run_vector());
                    results.push_back(run_indexed());
                }
            }
        }
    }
    return results;
}

struct CellKey {
    std::string suite;
    std::string type;
    std::string workload;
    std::size_t n = 0;
    friend auto operator<=>(const CellKey&, const CellKey&) = default;
};

struct CandidateKey {
    std::string structure;
    std::size_t leaf = 0;
    std::size_t fanout = 0;
    std::size_t levels = 0;
    friend auto operator<=>(const CandidateKey&, const CandidateKey&) = default;
};

double median(std::vector<double> values) {
    if (values.empty()) return std::numeric_limits<double>::quiet_NaN();
    std::sort(values.begin(), values.end());
    const auto middle = values.size() / 2;
    return values.size() % 2 == 0
        ? (values[middle - 1] + values[middle]) / 2.0
        : values[middle];
}

bool is_fixed_baseline(std::string_view structure) noexcept {
    return structure == "std::vector_reserved"
        || structure == "std::deque"
        || structure == "std::list_index_semantics"
        || structure == "std::vector_linear_lookup"
        || structure.starts_with("tiered_forced_leaf_");
}

void write_summary(const std::filesystem::path& path,
                   const std::vector<Result>& results) {
    using MeasurementMap = std::map<CandidateKey, std::map<CellKey, std::vector<double>>>;
    MeasurementMap measurements;
    for (const auto& result : results) {
        if (result.status != "ok" || result.total_ns <= 0) continue;
        CandidateKey candidate{result.structure, result.config.leaf_capacity,
                               result.config.directory_fanout,
                               result.config.directory_levels};
        CellKey cell{result.suite, result.type, result.workload, result.n};
        measurements[candidate][cell].push_back(result.total_ns);
    }

    std::map<CellKey, double> best;
    for (const auto& [candidate, cells] : measurements) {
        if (!is_fixed_baseline(candidate.structure)) continue;
        for (const auto& [cell, samples] : cells) {
            const auto value = median(samples);
            auto [iterator, inserted] = best.emplace(cell, value);
            if (!inserted) iterator->second = std::min(iterator->second, value);
        }
    }
    // A custom suite may intentionally contain no fixed baseline.  Keep its
    // summary usable without weakening the fixed-baseline rule elsewhere.
    if (best.empty()) {
        for (const auto& [candidate, cells] : measurements) {
            (void)candidate;
            for (const auto& [cell, samples] : cells) {
                const auto value = median(samples);
                auto [iterator, inserted] = best.emplace(cell, value);
                if (!inserted) iterator->second = std::min(iterator->second, value);
            }
        }
    }

    if (!path.parent_path().empty()) {
        std::filesystem::create_directories(path.parent_path());
    }
    std::ofstream output(path, std::ios::binary);
    if (!output) throw std::runtime_error("cannot open summary output: " + path.string());
    output << "profile,structure,leaf,fanout,levels,cells,geomean_vs_best_fixed,"
              "worst_cell_score,total_median_ns\n";
    output << std::setprecision(17);
    for (const auto& [candidate, cells] : measurements) {
        double log_sum = 0.0;
        double worst = std::numeric_limits<double>::infinity();
        double total = 0.0;
        std::size_t count = 0;
        for (const auto& [cell, samples] : cells) {
            const auto value = median(samples);
            const auto score = best[cell] / value;
            log_sum += std::log(score);
            worst = std::min(worst, score);
            total += value;
            ++count;
        }
        const auto geometric_mean = count == 0 ? 0.0 : std::exp(log_sum / count);
        output << csv_escape(UC_BUILD_PROFILE) << ',' << csv_escape(candidate.structure) << ','
               << candidate.leaf << ',' << candidate.fanout << ',' << candidate.levels << ','
               << count << ',' << geometric_mean << ',' << worst << ',' << total << '\n';
    }
}

Options parse_options(int argc, char** argv) {
    Options options;
    for (int i = 1; i < argc; ++i) {
        const std::string_view argument(argv[i]);
        auto require_value = [&]() -> std::string_view {
            if (++i >= argc) throw std::invalid_argument("missing value after " + std::string(argument));
            return argv[i];
        };
        if (argument == "--suite") options.suite = require_value();
        else if (argument == "--scale") options.scale = require_value();
        else if (argument == "--output") options.output = require_value();
        else if (argument == "--summary") options.summary = require_value();
        else if (argument == "--seed") options.seed = std::stoull(std::string(require_value()));
        else if (argument == "--leaf") options.leaf = std::stoull(std::string(require_value()));
        else if (argument == "--fanout") options.fanout = std::stoull(std::string(require_value()));
        else if (argument == "--levels") options.levels = std::stoull(std::string(require_value()));
        else if (argument == "--repeats") options.repeats_override = std::stoull(std::string(require_value()));
        else if (argument == "--n") options.n_override = std::stoull(std::string(require_value()));
        else if (argument == "--operations") options.operations_override = std::stoull(std::string(require_value()));
        else if (argument == "--edit-ppm") options.edit_ppm = std::stoull(std::string(require_value()));
        else if (argument == "--pattern") options.large_pattern = require_value();
        else if (argument == "--help") {
            std::cout << "uc_bench [--suite tune|compare|adapt|index|large|all] [--scale smoke|quick|full]\n"
                         "         [--output results.csv] [--summary summary.csv]\n"
                         "         [--leaf N] [--fanout N] [--levels N] [--seed N]\n"
                         "         large: --n N --operations N --edit-ppm 0..1000000\n"
                         "                --pattern steady_uniform|steady_localized|\n"
                         "                          bursty_uniform|bursty_localized|phase\n";
            std::exit(0);
        } else {
            throw std::invalid_argument("unknown argument: " + std::string(argument));
        }
    }
    if (options.suite != "tune" && options.suite != "compare"
        && options.suite != "adapt"
        && options.suite != "index" && options.suite != "large"
        && options.suite != "all") {
        throw std::invalid_argument(
            "suite must be tune, compare, adapt, index, large, or all");
    }
    if (options.scale != "smoke" && options.scale != "quick" && options.scale != "full") {
        throw std::invalid_argument("scale must be smoke, quick, or full");
    }
    if (options.edit_ppm > 1'000'000) {
        throw std::invalid_argument("edit ppm must be in 0..1000000");
    }
    if (options.suite == "large"
        && (options.n_override == 0 || options.operations_override == 0)) {
        throw std::invalid_argument(
            "large suite requires --n and --operations");
    }
    if (options.suite == "large") {
        constexpr std::size_t maximum_large_operations = 8'000'000;
        if (options.operations_override > maximum_large_operations) {
            throw std::invalid_argument(
                "large suite supports at most 8000000 operations per process");
        }
        if (options.n_override
            > static_cast<std::size_t>(std::numeric_limits<std::ptrdiff_t>::max())) {
            throw std::invalid_argument("large n exceeds PTRDIFF_MAX");
        }
        (void)initial_reserve_capacity(options.n_override);
    }
    return options;
}

void print_environment(const Options& options) {
    std::cout << "UniversalContainer benchmark\n"
              << "  profile: " << UC_BUILD_PROFILE << "\n"
              << "  suite: " << options.suite << "\n"
              << "  scale: " << options.scale << "\n"
              << "  hardware threads: " << std::thread::hardware_concurrency() << "\n"
              << "  steady_clock period: " << Clock::period::num << '/' << Clock::period::den << "\n"
              << "  default tiered config: leaf=" << options.leaf
              << " fanout=" << options.fanout << " levels=" << options.levels << "\n";
    if (options.suite == "large") {
        std::cout << "  large cell: n=" << options.n_override
                  << " operations=" << options.operations_override
                  << " edit_ppm=" << options.edit_ppm
                  << " pattern=" << options.large_pattern << "\n";
    }
#ifdef _MSC_FULL_VER
    std::cout << "  MSVC full version: " << _MSC_FULL_VER << "\n";
#endif
}

} // namespace

int main(int argc, char** argv) {
    try {
        const auto options = parse_options(argc, argv);
        print_environment(options);

        std::vector<Result> results;
        if (options.suite == "tune" || options.suite == "all") {
            auto tuning = run_tuning(options);
            results.insert(results.end(), std::make_move_iterator(tuning.begin()),
                           std::make_move_iterator(tuning.end()));
        }
        if (options.suite == "compare" || options.suite == "all") {
            auto comparison = run_comparison(options);
            results.insert(results.end(), std::make_move_iterator(comparison.begin()),
                           std::make_move_iterator(comparison.end()));
        }
        if (options.suite == "adapt" || options.suite == "all") {
            auto adaptation = run_adaptation_benchmark(options);
            results.insert(results.end(), std::make_move_iterator(adaptation.begin()),
                           std::make_move_iterator(adaptation.end()));
        }
        if (options.suite == "index" || options.suite == "all") {
            auto indexed = run_index_benchmark(options);
            results.insert(results.end(), std::make_move_iterator(indexed.begin()),
                           std::make_move_iterator(indexed.end()));
        }
        if (options.suite == "large") {
            auto large = run_large_cell(options);
            results.insert(results.end(), std::make_move_iterator(large.begin()),
                           std::make_move_iterator(large.end()));
        }

        write_results(options.output, results);
        write_summary(options.summary, results);
        std::cout << "wrote " << results.size() << " raw rows to " << options.output << '\n'
                  << "wrote aggregate scores to " << options.summary << '\n';
        return 0;
    } catch (const std::exception& error) {
        std::cerr << "benchmark error: " << error.what() << '\n';
        return 1;
    }
}
