Realisation v1
This commit is contained in:
Efim Beshmenev
@@ -0,0 +1,732 @@
|
||||
#pragma once
|
||||
|
||||
#include "universal_container/tiered_storage.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
namespace uc {
|
||||
|
||||
enum class StorageMode : std::uint8_t {
|
||||
vector,
|
||||
tiered
|
||||
};
|
||||
|
||||
enum class ResidencyMode : std::uint8_t {
|
||||
automatic,
|
||||
forced_vector,
|
||||
forced_tiered
|
||||
};
|
||||
|
||||
// Deferred is the safe default: reads collect statistics, but a representation
|
||||
// change is performed only at a mutation boundary or by adapt_now(). Eager is
|
||||
// retained as an explicitly unsafe experiment for non-const reads.
|
||||
enum class ReadAdaptationMode : std::uint8_t {
|
||||
deferred,
|
||||
eager_nonconst
|
||||
};
|
||||
|
||||
enum class OperationKind : std::uint8_t {
|
||||
random_read,
|
||||
sequential_read,
|
||||
append,
|
||||
insert,
|
||||
erase,
|
||||
set
|
||||
};
|
||||
|
||||
struct OperationSample {
|
||||
OperationKind kind = OperationKind::random_read;
|
||||
std::size_t size = 0;
|
||||
std::size_t position = 0;
|
||||
std::size_t count = 1;
|
||||
std::size_t element_bytes = 4;
|
||||
};
|
||||
|
||||
struct AdaptationDecision {
|
||||
StorageMode target = StorageMode::vector;
|
||||
TieredConfig tiered_config{};
|
||||
double expected_saving = 0.0;
|
||||
};
|
||||
|
||||
struct AdaptationConfig {
|
||||
// Selected by the paired aggressive/moderate/balanced benchmark: this
|
||||
// reacts early in a sustained edit phase but leaves a 128-edit negative
|
||||
// control in vector mode.
|
||||
std::size_t evaluation_interval = 256;
|
||||
std::size_t minimum_observations = 2048;
|
||||
std::size_t edit_evaluation_interval = 32;
|
||||
std::size_t minimum_edit_observations = 128;
|
||||
std::size_t minimum_tiered_size = 8 * 1024;
|
||||
// The forecast is deliberately bounded by both evidence accumulated in
|
||||
// the current residency and this hard ceiling. Treating the ceiling as a
|
||||
// guaranteed future phase length makes short edit bursts look much more
|
||||
// profitable than they really are.
|
||||
std::size_t forecast_operations = 128 * 1024;
|
||||
std::size_t minimum_forecast_operations = 256;
|
||||
double forecast_growth_factor = 8.0;
|
||||
// A sampled read carries `count = read_sample_rate`, so the statistical
|
||||
// weight is preserved while the hot operator[] path pays the full policy
|
||||
// update only once per cache-line-sized batch of accesses.
|
||||
std::size_t read_sample_rate = 256;
|
||||
// Structural samples retain their aggregate weight. Sampling every edit
|
||||
// made the replaceable policy itself a visible part of edit latency.
|
||||
std::size_t edit_sample_rate = 4;
|
||||
std::size_t minimum_residency_operations = 256;
|
||||
// Storage-mode hysteresis uses the edit share of the workload EWMA. The
|
||||
// wider entry threshold and narrower exit threshold form a Schmitt trigger:
|
||||
// a tiered sequence is not converted back to vector while edits are still
|
||||
// a material part of the current phase.
|
||||
double tiered_entry_edit_fraction = 0.05;
|
||||
double tiered_exit_edit_fraction = 0.01;
|
||||
std::size_t required_confirmation_windows = 2;
|
||||
// Shape changes are independent from storage-mode changes. They need a
|
||||
// longer residency and an edit-heavy phase because changing leaf capacity
|
||||
// is itself a full O(N) rebuild.
|
||||
std::size_t minimum_shape_residency_operations = 4 * 1024;
|
||||
double minimum_shape_edit_fraction = 0.10;
|
||||
std::size_t locality_radius_divisor = 50;
|
||||
std::size_t maximum_directory_levels = 4;
|
||||
double vector_to_tiered_safety = 1.50;
|
||||
double tiered_to_vector_safety = 2.00;
|
||||
double tiered_rebuild_safety = 2.50;
|
||||
// A leaf rebuild needs a material advantage over the incumbent, not merely
|
||||
// to win a noisy window. Directory-only changes still have their much
|
||||
// smaller rebuild cost, but use the same relative stability guard.
|
||||
double minimum_shape_improvement = 0.15;
|
||||
double ewma_alpha = 0.35;
|
||||
|
||||
// The search space is data, not a hard-coded branch in the container. A
|
||||
// different policy/config can replace it without touching storage code.
|
||||
std::array<std::size_t, 5> tiered_leaf_candidates{64, 128, 256, 512, 1024};
|
||||
|
||||
// Calibratable cost units. The leaf movement term grows with B, while the
|
||||
// directory/fragmentation term grows with N/B. Their intersection is why
|
||||
// the selected block size changes with both N and edit locality.
|
||||
double vector_read = 1.0;
|
||||
double tiered_read_base = 12.0;
|
||||
double tiered_read_per_level = 2.0;
|
||||
double tiered_lookup_per_log_top = 1.20;
|
||||
double vector_scan_element = 0.70;
|
||||
double tiered_scan_element = 0.92;
|
||||
double tiered_scan_boundary = 12.0;
|
||||
double vector_append = 2.0;
|
||||
double tiered_append = 2.8;
|
||||
double vector_edit_fixed = 6.0;
|
||||
double tiered_edit_fixed = 30.0;
|
||||
double vector_move_unit = 0.025;
|
||||
double tiered_move_unit = 4.0;
|
||||
double tiered_directory_unit = 0.65;
|
||||
double localized_directory_multiplier = 0.15;
|
||||
double conversion_unit_per_element = 1.0;
|
||||
double directory_rebuild_unit_per_leaf = 8.0;
|
||||
};
|
||||
|
||||
struct AdaptationTelemetry {
|
||||
std::uint64_t observed_operations = 0;
|
||||
std::uint64_t sampled_reads = 0;
|
||||
std::uint64_t evaluations = 0;
|
||||
std::uint64_t vector_to_tiered = 0;
|
||||
std::uint64_t tiered_to_vector = 0;
|
||||
std::uint64_t tiered_rebuilds = 0;
|
||||
std::uint64_t tiered_leaf_rebuilds = 0;
|
||||
std::uint64_t tiered_directory_rebuilds = 0;
|
||||
std::size_t last_recommended_leaf = 0;
|
||||
std::size_t last_recommended_levels = 0;
|
||||
double last_vector_cost_per_operation = 0.0;
|
||||
double last_tiered_cost_per_operation = 0.0;
|
||||
double last_current_cost_per_operation = 0.0;
|
||||
double last_expected_saving = 0.0;
|
||||
double last_conversion_cost = 0.0;
|
||||
double last_local_edit_fraction = 0.0;
|
||||
double last_edit_fraction = 0.0;
|
||||
std::size_t last_forecast_operations = 0;
|
||||
std::size_t last_evidence_windows = 0;
|
||||
};
|
||||
|
||||
class CostModelPolicy {
|
||||
public:
|
||||
static constexpr std::size_t candidate_count = 5;
|
||||
|
||||
explicit CostModelPolicy(AdaptationConfig config = {}, TieredConfig tiered = {})
|
||||
: config_(normalize(config)), active_tiered_(normalize_tiered(tiered)),
|
||||
search_fanout_(active_tiered_.directory_fanout),
|
||||
search_max_levels_(config_.maximum_directory_levels) {}
|
||||
|
||||
void set_tiered_config(TieredConfig tiered) noexcept {
|
||||
active_tiered_ = normalize_tiered(tiered);
|
||||
search_fanout_ = active_tiered_.directory_fanout;
|
||||
search_max_levels_ = config_.maximum_directory_levels;
|
||||
}
|
||||
|
||||
void observe(const OperationSample& sample) noexcept {
|
||||
const auto count = static_cast<double>(std::max<std::size_t>(1, sample.count));
|
||||
last_element_bytes_ = std::max<std::size_t>(1, sample.element_bytes);
|
||||
last_observed_size_ = sample.size;
|
||||
operations_since_transition_ += sample.count;
|
||||
operations_since_shape_transition_ += sample.count;
|
||||
telemetry_.observed_operations += sample.count;
|
||||
if (sample.kind == OperationKind::random_read) {
|
||||
++telemetry_.sampled_reads;
|
||||
}
|
||||
|
||||
const bool edit = is_edit(sample.kind);
|
||||
// In vector mode, reads/appends cannot make tiered storage preferable.
|
||||
// Before the first relevant edit (or below the measured small-size
|
||||
// threshold), avoid evaluating all five counterfactual shapes in the
|
||||
// hot path. Once edits exist, sampled reads are evaluated normally so
|
||||
// that a short burst can be rejected rather than converted later.
|
||||
if (active_mode_ == StorageMode::vector
|
||||
&& (sample.size < config_.minimum_tiered_size
|
||||
|| (!edit && total_edit_weight_ == 0.0
|
||||
&& window_edit_weight_ == 0.0
|
||||
&& !decision_ready_ && !pending_decision_))) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bool localized = classify_edit_locality(sample);
|
||||
|
||||
window_vector_cost_ += estimate_vector(sample);
|
||||
window_active_tiered_cost_ += estimate_tiered(sample, active_tiered_, localized);
|
||||
for (std::size_t i = 0; i < candidate_count; ++i) {
|
||||
window_candidate_costs_[i] += estimate_tiered(
|
||||
sample, candidate_config(i, sample.size), localized);
|
||||
}
|
||||
window_weight_ += count;
|
||||
if (edit) {
|
||||
window_edit_weight_ += count;
|
||||
total_edit_weight_ += count;
|
||||
if (localized) {
|
||||
window_local_edit_weight_ += count;
|
||||
}
|
||||
}
|
||||
|
||||
if (window_weight_ >= static_cast<double>(config_.evaluation_interval)
|
||||
|| window_edit_weight_ >= static_cast<double>(
|
||||
config_.edit_evaluation_interval)) {
|
||||
close_window();
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] bool decision_ready() const noexcept { return decision_ready_; }
|
||||
|
||||
[[nodiscard]] std::optional<AdaptationDecision>
|
||||
recommended_decision(StorageMode current,
|
||||
std::size_t size,
|
||||
TieredConfig current_config) noexcept {
|
||||
active_mode_ = current;
|
||||
if (!decision_ready_) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const auto evidence_windows = closed_windows_since_decision_;
|
||||
decision_ready_ = false;
|
||||
closed_windows_since_decision_ = 0;
|
||||
++telemetry_.evaluations;
|
||||
telemetry_.last_evidence_windows = evidence_windows;
|
||||
const auto evaluation = evaluate_decision(current, size, current_config);
|
||||
telemetry_.last_vector_cost_per_operation = evaluation.vector_cost;
|
||||
telemetry_.last_tiered_cost_per_operation = evaluation.best_tiered_cost;
|
||||
telemetry_.last_current_cost_per_operation = evaluation.current_cost;
|
||||
telemetry_.last_recommended_leaf = evaluation.best_config.leaf_capacity;
|
||||
telemetry_.last_recommended_levels = evaluation.best_config.directory_levels;
|
||||
telemetry_.last_conversion_cost = evaluation.selected_rebuild_cost;
|
||||
telemetry_.last_expected_saving = evaluation.decision
|
||||
? evaluation.decision->expected_saving : 0.0;
|
||||
telemetry_.last_forecast_operations = evaluation.forecast_operations;
|
||||
telemetry_.last_edit_fraction = ewma_edit_fraction_;
|
||||
|
||||
return confirm(evaluation.decision);
|
||||
}
|
||||
|
||||
// Compatibility with simpler storage wrappers: this intentionally loses a
|
||||
// same-mode tiered shape recommendation.
|
||||
[[nodiscard]] std::optional<StorageMode>
|
||||
recommended_mode(StorageMode current, std::size_t size) noexcept {
|
||||
const auto decision = recommended_decision(current, size, active_tiered_);
|
||||
return decision ? std::optional<StorageMode>(decision->target) : std::nullopt;
|
||||
}
|
||||
|
||||
void on_transition(StorageMode from,
|
||||
StorageMode to,
|
||||
TieredConfig active_config) noexcept {
|
||||
if (from == StorageMode::vector && to == StorageMode::tiered) {
|
||||
++telemetry_.vector_to_tiered;
|
||||
} else if (from == StorageMode::tiered && to == StorageMode::vector) {
|
||||
++telemetry_.tiered_to_vector;
|
||||
} else if (from == StorageMode::tiered && to == StorageMode::tiered) {
|
||||
++telemetry_.tiered_rebuilds;
|
||||
if (normalize_tiered(active_config).leaf_capacity
|
||||
== active_tiered_.leaf_capacity) {
|
||||
++telemetry_.tiered_directory_rebuilds;
|
||||
} else {
|
||||
++telemetry_.tiered_leaf_rebuilds;
|
||||
}
|
||||
}
|
||||
const auto normalized_active = normalize_tiered(active_config);
|
||||
if (to == StorageMode::tiered) {
|
||||
synchronize_active_tiered_ewma(normalized_active);
|
||||
}
|
||||
active_tiered_ = normalized_active;
|
||||
active_mode_ = to;
|
||||
clear_transition_evidence();
|
||||
}
|
||||
|
||||
void on_transition(StorageMode from, StorageMode to) noexcept {
|
||||
on_transition(from, to, active_tiered_);
|
||||
}
|
||||
|
||||
void reset() noexcept {
|
||||
const auto active = active_tiered_;
|
||||
const auto mode = active_mode_;
|
||||
const auto fanout = search_fanout_;
|
||||
const auto levels = search_max_levels_;
|
||||
*this = CostModelPolicy(config_, active);
|
||||
active_mode_ = mode;
|
||||
search_fanout_ = fanout;
|
||||
search_max_levels_ = levels;
|
||||
}
|
||||
|
||||
[[nodiscard]] const AdaptationConfig& config() const noexcept { return config_; }
|
||||
[[nodiscard]] const AdaptationTelemetry& telemetry() const noexcept { return telemetry_; }
|
||||
|
||||
private:
|
||||
static bool is_edit(OperationKind kind) noexcept {
|
||||
return kind == OperationKind::insert || kind == OperationKind::erase;
|
||||
}
|
||||
|
||||
static TieredConfig normalize_tiered(TieredConfig config) noexcept {
|
||||
config.leaf_capacity = std::clamp<std::size_t>(config.leaf_capacity, 4, 1U << 20U);
|
||||
config.directory_fanout = std::clamp<std::size_t>(
|
||||
config.directory_fanout, 2, 1U << 16U);
|
||||
config.directory_levels = std::clamp<std::size_t>(config.directory_levels, 1, 8);
|
||||
return config;
|
||||
}
|
||||
|
||||
static AdaptationConfig normalize(AdaptationConfig config) noexcept {
|
||||
config.evaluation_interval = std::max<std::size_t>(16, config.evaluation_interval);
|
||||
config.minimum_observations = std::max(config.evaluation_interval,
|
||||
config.minimum_observations);
|
||||
config.edit_evaluation_interval = std::max<std::size_t>(8,
|
||||
config.edit_evaluation_interval);
|
||||
config.minimum_edit_observations = std::max(
|
||||
config.edit_evaluation_interval, config.minimum_edit_observations);
|
||||
config.forecast_operations = std::max<std::size_t>(1, config.forecast_operations);
|
||||
config.minimum_forecast_operations = std::clamp<std::size_t>(
|
||||
config.minimum_forecast_operations, 1, config.forecast_operations);
|
||||
config.forecast_growth_factor = std::max(1.0, config.forecast_growth_factor);
|
||||
config.read_sample_rate = std::max<std::size_t>(1, config.read_sample_rate);
|
||||
config.edit_sample_rate = std::max<std::size_t>(1, config.edit_sample_rate);
|
||||
config.tiered_entry_edit_fraction = std::clamp(
|
||||
config.tiered_entry_edit_fraction, 0.0, 1.0);
|
||||
config.tiered_exit_edit_fraction = std::clamp(
|
||||
config.tiered_exit_edit_fraction, 0.0,
|
||||
config.tiered_entry_edit_fraction);
|
||||
config.required_confirmation_windows = std::max<std::size_t>(
|
||||
1, config.required_confirmation_windows);
|
||||
config.minimum_shape_edit_fraction = std::clamp(
|
||||
config.minimum_shape_edit_fraction, 0.0, 1.0);
|
||||
config.locality_radius_divisor = std::max<std::size_t>(1,
|
||||
config.locality_radius_divisor);
|
||||
config.maximum_directory_levels = std::clamp<std::size_t>(
|
||||
config.maximum_directory_levels, 1, 8);
|
||||
config.minimum_shape_improvement = std::clamp(
|
||||
config.minimum_shape_improvement, 0.0, 1.0);
|
||||
config.ewma_alpha = std::clamp(config.ewma_alpha, 0.01, 1.0);
|
||||
for (auto& leaf : config.tiered_leaf_candidates) {
|
||||
leaf = std::clamp<std::size_t>(leaf, 4, 1U << 20U);
|
||||
}
|
||||
std::sort(config.tiered_leaf_candidates.begin(),
|
||||
config.tiered_leaf_candidates.end());
|
||||
return config;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t actual_levels(std::size_t size,
|
||||
std::size_t leaf,
|
||||
std::size_t fanout,
|
||||
std::size_t maximum) const noexcept {
|
||||
auto nodes = (size + leaf - 1) / leaf;
|
||||
std::size_t levels = 1;
|
||||
while (nodes > fanout && levels < maximum) {
|
||||
nodes = (nodes + fanout - 1) / fanout;
|
||||
++levels;
|
||||
}
|
||||
return levels;
|
||||
}
|
||||
|
||||
[[nodiscard]] TieredConfig candidate_config(std::size_t index,
|
||||
std::size_t size) const noexcept {
|
||||
const auto leaf = config_.tiered_leaf_candidates[index];
|
||||
return {leaf, search_fanout_,
|
||||
actual_levels(size, leaf, search_fanout_, search_max_levels_)};
|
||||
}
|
||||
|
||||
bool classify_edit_locality(const OperationSample& sample) noexcept {
|
||||
if (!is_edit(sample.kind)) {
|
||||
return false;
|
||||
}
|
||||
bool localized = false;
|
||||
if (has_last_edit_) {
|
||||
const auto distance = sample.position > last_edit_position_
|
||||
? sample.position - last_edit_position_
|
||||
: last_edit_position_ - sample.position;
|
||||
const auto radius = std::max<std::size_t>(
|
||||
32, sample.size / config_.locality_radius_divisor);
|
||||
localized = distance <= radius;
|
||||
}
|
||||
last_edit_position_ = sample.position;
|
||||
has_last_edit_ = true;
|
||||
return localized;
|
||||
}
|
||||
|
||||
[[nodiscard]] double estimate_vector(const OperationSample& sample) const noexcept {
|
||||
const auto count = static_cast<double>(std::max<std::size_t>(1, sample.count));
|
||||
switch (sample.kind) {
|
||||
case OperationKind::random_read:
|
||||
return config_.vector_read * count;
|
||||
case OperationKind::sequential_read:
|
||||
return config_.vector_scan_element * count;
|
||||
case OperationKind::append:
|
||||
return config_.vector_append * count;
|
||||
case OperationKind::set:
|
||||
return config_.vector_read * 1.3 * count;
|
||||
case OperationKind::insert:
|
||||
case OperationKind::erase: {
|
||||
const auto width_scale = std::max(1.0,
|
||||
static_cast<double>(sample.element_bytes) / sizeof(std::uint32_t));
|
||||
const auto tail = sample.position < sample.size
|
||||
? sample.size - sample.position : 0;
|
||||
return (config_.vector_edit_fixed
|
||||
+ config_.vector_move_unit * width_scale * static_cast<double>(tail))
|
||||
* count;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
[[nodiscard]] double estimate_tiered(const OperationSample& sample,
|
||||
TieredConfig tiered,
|
||||
bool localized) const noexcept {
|
||||
tiered = normalize_tiered(tiered);
|
||||
const auto count = static_cast<double>(std::max<std::size_t>(1, sample.count));
|
||||
const auto leaf_count = std::max<std::size_t>(
|
||||
1, (sample.size + tiered.leaf_capacity - 1) / tiered.leaf_capacity);
|
||||
const auto level_count = actual_levels(
|
||||
sample.size, tiered.leaf_capacity, tiered.directory_fanout,
|
||||
tiered.directory_levels);
|
||||
auto top_nodes = leaf_count;
|
||||
for (std::size_t level = 1; level < level_count; ++level) {
|
||||
top_nodes = (top_nodes + tiered.directory_fanout - 1)
|
||||
/ tiered.directory_fanout;
|
||||
}
|
||||
const auto levels = static_cast<double>(level_count);
|
||||
const auto top_log2 = top_nodes <= 1 ? 0.0 : static_cast<double>(
|
||||
std::bit_width(top_nodes) - 1U);
|
||||
const auto lookup = config_.tiered_read_base
|
||||
+ config_.tiered_read_per_level * levels
|
||||
+ config_.tiered_lookup_per_log_top * top_log2;
|
||||
switch (sample.kind) {
|
||||
case OperationKind::random_read:
|
||||
return lookup * count;
|
||||
case OperationKind::sequential_read:
|
||||
return (config_.tiered_scan_element
|
||||
+ config_.tiered_scan_boundary
|
||||
/ static_cast<double>(tiered.leaf_capacity)) * count;
|
||||
case OperationKind::append:
|
||||
return (config_.tiered_append
|
||||
+ 8.0 / static_cast<double>(tiered.leaf_capacity)) * count;
|
||||
case OperationKind::set:
|
||||
return lookup * 1.3 * count;
|
||||
case OperationKind::insert:
|
||||
case OperationKind::erase: {
|
||||
const auto width_scale = std::max(1.0,
|
||||
static_cast<double>(sample.element_bytes) / sizeof(std::uint32_t));
|
||||
const auto local = sample.position % tiered.leaf_capacity;
|
||||
const auto local_moves = std::min(local, tiered.leaf_capacity - local);
|
||||
const auto directory_multiplier = localized
|
||||
? config_.localized_directory_multiplier : 1.0;
|
||||
const auto directory_work = config_.tiered_directory_unit
|
||||
* directory_multiplier
|
||||
* static_cast<double>(leaf_count);
|
||||
return (config_.tiered_edit_fixed
|
||||
+ config_.tiered_move_unit * width_scale
|
||||
* static_cast<double>(local_moves)
|
||||
+ directory_work + lookup) * count;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
struct DecisionEvaluation {
|
||||
std::optional<AdaptationDecision> decision;
|
||||
TieredConfig best_config{};
|
||||
double vector_cost = 0.0;
|
||||
double best_tiered_cost = 0.0;
|
||||
double current_cost = 0.0;
|
||||
double selected_rebuild_cost = 0.0;
|
||||
std::size_t forecast_operations = 0;
|
||||
};
|
||||
|
||||
[[nodiscard]] DecisionEvaluation evaluate_decision(
|
||||
StorageMode current,
|
||||
std::size_t size,
|
||||
TieredConfig current_config) const noexcept {
|
||||
DecisionEvaluation evaluation;
|
||||
|
||||
std::size_t best_index = 0;
|
||||
for (std::size_t i = 1; i < candidate_count; ++i) {
|
||||
if (ewma_candidate_costs_[i] < ewma_candidate_costs_[best_index]) {
|
||||
best_index = i;
|
||||
}
|
||||
}
|
||||
evaluation.best_config = candidate_config(best_index, size);
|
||||
evaluation.vector_cost = ewma_vector_cost_;
|
||||
evaluation.best_tiered_cost = ewma_candidate_costs_[best_index];
|
||||
const auto current_tiered_cost = ewma_active_tiered_cost_;
|
||||
evaluation.current_cost = current == StorageMode::vector
|
||||
? evaluation.vector_cost : current_tiered_cost;
|
||||
|
||||
const auto width_scale = std::max(1.0,
|
||||
static_cast<double>(last_element_bytes_) / sizeof(std::uint32_t));
|
||||
const auto conversion_cost = static_cast<double>(size)
|
||||
* config_.conversion_unit_per_element * width_scale;
|
||||
evaluation.selected_rebuild_cost = conversion_cost;
|
||||
const auto evidence_forecast = static_cast<std::size_t>(std::ceil(
|
||||
total_weight_ * config_.forecast_growth_factor));
|
||||
evaluation.forecast_operations = std::clamp(
|
||||
evidence_forecast,
|
||||
config_.minimum_forecast_operations,
|
||||
config_.forecast_operations);
|
||||
|
||||
const bool enough_evidence =
|
||||
total_weight_ >= static_cast<double>(config_.minimum_observations)
|
||||
|| total_edit_weight_
|
||||
>= static_cast<double>(config_.minimum_edit_observations);
|
||||
if (!initialized_ || !enough_evidence
|
||||
|| operations_since_transition_ < config_.minimum_residency_operations) {
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
const auto forecast = static_cast<double>(evaluation.forecast_operations);
|
||||
double best_net_saving = 0.0;
|
||||
|
||||
if (current == StorageMode::vector) {
|
||||
if (size >= config_.minimum_tiered_size
|
||||
&& ewma_edit_fraction_ >= config_.tiered_entry_edit_fraction
|
||||
&& evaluation.best_tiered_cost < evaluation.vector_cost) {
|
||||
const auto gross = (evaluation.vector_cost
|
||||
- evaluation.best_tiered_cost) * forecast;
|
||||
const auto threshold = conversion_cost
|
||||
* config_.vector_to_tiered_safety;
|
||||
if (gross > threshold) {
|
||||
evaluation.decision = AdaptationDecision{
|
||||
StorageMode::tiered, evaluation.best_config, gross};
|
||||
best_net_saving = gross - threshold;
|
||||
}
|
||||
}
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
// A tiered sequence returns to vector only after the workload has
|
||||
// crossed the low (read-dominant) side of the phase hysteresis band.
|
||||
if (ewma_edit_fraction_ <= config_.tiered_exit_edit_fraction
|
||||
&& evaluation.vector_cost < current_tiered_cost) {
|
||||
const auto gross = (current_tiered_cost - evaluation.vector_cost) * forecast;
|
||||
const auto threshold = conversion_cost * config_.tiered_to_vector_safety;
|
||||
if (gross > threshold) {
|
||||
evaluation.decision = AdaptationDecision{
|
||||
StorageMode::vector, normalize_tiered(current_config), gross};
|
||||
best_net_saving = gross - threshold;
|
||||
}
|
||||
}
|
||||
|
||||
const auto normalized_current = normalize_tiered(current_config);
|
||||
const bool shape_differs = evaluation.best_config != normalized_current;
|
||||
const bool directory_only = evaluation.best_config.leaf_capacity
|
||||
== normalized_current.leaf_capacity;
|
||||
const auto shape_rebuild_cost = directory_only
|
||||
? static_cast<double>((size + evaluation.best_config.leaf_capacity - 1)
|
||||
/ evaluation.best_config.leaf_capacity)
|
||||
* config_.directory_rebuild_unit_per_leaf
|
||||
: conversion_cost;
|
||||
const auto relative_improvement = current_tiered_cost > 0.0
|
||||
? (current_tiered_cost - evaluation.best_tiered_cost)
|
||||
/ current_tiered_cost
|
||||
: 0.0;
|
||||
if (operations_since_shape_transition_
|
||||
>= config_.minimum_shape_residency_operations
|
||||
&& ewma_edit_fraction_ >= config_.minimum_shape_edit_fraction
|
||||
&& shape_differs
|
||||
&& evaluation.best_tiered_cost < current_tiered_cost
|
||||
&& relative_improvement >= config_.minimum_shape_improvement) {
|
||||
const auto gross = (current_tiered_cost
|
||||
- evaluation.best_tiered_cost) * forecast;
|
||||
const auto threshold = shape_rebuild_cost * config_.tiered_rebuild_safety;
|
||||
const auto net = gross - threshold;
|
||||
if (gross > threshold && net > best_net_saving) {
|
||||
evaluation.decision = AdaptationDecision{
|
||||
StorageMode::tiered, evaluation.best_config, gross};
|
||||
evaluation.selected_rebuild_cost = shape_rebuild_cost;
|
||||
}
|
||||
}
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
void close_window() noexcept {
|
||||
const auto inverse_weight = 1.0 / window_weight_;
|
||||
const auto vector_per_operation = window_vector_cost_ * inverse_weight;
|
||||
const auto active_per_operation = window_active_tiered_cost_ * inverse_weight;
|
||||
const auto edit_fraction = window_edit_weight_ * inverse_weight;
|
||||
std::array<double, candidate_count> candidates{};
|
||||
for (std::size_t i = 0; i < candidate_count; ++i) {
|
||||
candidates[i] = window_candidate_costs_[i] * inverse_weight;
|
||||
}
|
||||
if (!initialized_) {
|
||||
ewma_vector_cost_ = vector_per_operation;
|
||||
ewma_active_tiered_cost_ = active_per_operation;
|
||||
ewma_candidate_costs_ = candidates;
|
||||
ewma_edit_fraction_ = edit_fraction;
|
||||
initialized_ = true;
|
||||
} else {
|
||||
const auto alpha = config_.ewma_alpha;
|
||||
ewma_vector_cost_ = alpha * vector_per_operation
|
||||
+ (1.0 - alpha) * ewma_vector_cost_;
|
||||
ewma_active_tiered_cost_ = alpha * active_per_operation
|
||||
+ (1.0 - alpha) * ewma_active_tiered_cost_;
|
||||
for (std::size_t i = 0; i < candidate_count; ++i) {
|
||||
ewma_candidate_costs_[i] = alpha * candidates[i]
|
||||
+ (1.0 - alpha) * ewma_candidate_costs_[i];
|
||||
}
|
||||
ewma_edit_fraction_ = alpha * edit_fraction
|
||||
+ (1.0 - alpha) * ewma_edit_fraction_;
|
||||
}
|
||||
if (window_edit_weight_ > 0.0) {
|
||||
telemetry_.last_local_edit_fraction =
|
||||
window_local_edit_weight_ / window_edit_weight_;
|
||||
}
|
||||
telemetry_.last_edit_fraction = ewma_edit_fraction_;
|
||||
total_weight_ += window_weight_;
|
||||
window_vector_cost_ = 0.0;
|
||||
window_active_tiered_cost_ = 0.0;
|
||||
window_candidate_costs_.fill(0.0);
|
||||
window_weight_ = 0.0;
|
||||
window_edit_weight_ = 0.0;
|
||||
window_local_edit_weight_ = 0.0;
|
||||
update_confirmation(evaluate_decision(
|
||||
active_mode_, last_observed_size_, active_tiered_).decision);
|
||||
decision_ready_ = true;
|
||||
if (closed_windows_since_decision_
|
||||
!= std::numeric_limits<std::size_t>::max()) {
|
||||
++closed_windows_since_decision_;
|
||||
}
|
||||
}
|
||||
|
||||
static bool same_decision(const AdaptationDecision& left,
|
||||
const AdaptationDecision& right) noexcept {
|
||||
return left.target == right.target
|
||||
&& left.tiered_config == right.tiered_config;
|
||||
}
|
||||
|
||||
void update_confirmation(
|
||||
const std::optional<AdaptationDecision>& decision) noexcept {
|
||||
if (!decision) {
|
||||
pending_decision_.reset();
|
||||
confirmation_count_ = 0;
|
||||
return;
|
||||
}
|
||||
if (pending_decision_ && same_decision(*pending_decision_, *decision)) {
|
||||
confirmation_count_ = std::min(
|
||||
config_.required_confirmation_windows,
|
||||
confirmation_count_ + 1);
|
||||
pending_decision_ = decision;
|
||||
} else {
|
||||
pending_decision_ = decision;
|
||||
confirmation_count_ = 1;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] std::optional<AdaptationDecision>
|
||||
confirm(const std::optional<AdaptationDecision>& decision) noexcept {
|
||||
if (!decision || !pending_decision_
|
||||
|| !same_decision(*pending_decision_, *decision)
|
||||
|| confirmation_count_ < config_.required_confirmation_windows) {
|
||||
return std::nullopt;
|
||||
}
|
||||
auto confirmed = decision;
|
||||
pending_decision_.reset();
|
||||
confirmation_count_ = 0;
|
||||
return confirmed;
|
||||
}
|
||||
|
||||
void synchronize_active_tiered_ewma(TieredConfig active) noexcept {
|
||||
if (!initialized_) {
|
||||
return;
|
||||
}
|
||||
for (std::size_t i = 0; i < candidate_count; ++i) {
|
||||
if (config_.tiered_leaf_candidates[i] == active.leaf_capacity) {
|
||||
ewma_active_tiered_cost_ = ewma_candidate_costs_[i];
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A committed transition starts a new payback/residency epoch, but the
|
||||
// workload signal is deliberately retained. Throwing the EWMA away made
|
||||
// the policy relearn the same stationary phase from noisy short windows and
|
||||
// allowed vector<->tiered and leaf-shape ping-pong.
|
||||
void clear_transition_evidence() noexcept {
|
||||
operations_since_transition_ = 0;
|
||||
operations_since_shape_transition_ = 0;
|
||||
total_weight_ = 0.0;
|
||||
total_edit_weight_ = 0.0;
|
||||
window_weight_ = 0.0;
|
||||
window_vector_cost_ = 0.0;
|
||||
window_active_tiered_cost_ = 0.0;
|
||||
window_candidate_costs_.fill(0.0);
|
||||
window_edit_weight_ = 0.0;
|
||||
window_local_edit_weight_ = 0.0;
|
||||
pending_decision_.reset();
|
||||
confirmation_count_ = 0;
|
||||
has_last_edit_ = false;
|
||||
decision_ready_ = false;
|
||||
closed_windows_since_decision_ = 0;
|
||||
}
|
||||
|
||||
AdaptationConfig config_;
|
||||
TieredConfig active_tiered_;
|
||||
StorageMode active_mode_ = StorageMode::vector;
|
||||
std::size_t search_fanout_ = 64;
|
||||
std::size_t search_max_levels_ = 4;
|
||||
AdaptationTelemetry telemetry_;
|
||||
std::array<double, candidate_count> window_candidate_costs_{};
|
||||
std::array<double, candidate_count> ewma_candidate_costs_{};
|
||||
double window_vector_cost_ = 0.0;
|
||||
double window_active_tiered_cost_ = 0.0;
|
||||
double window_weight_ = 0.0;
|
||||
double window_edit_weight_ = 0.0;
|
||||
double window_local_edit_weight_ = 0.0;
|
||||
double total_weight_ = 0.0;
|
||||
double total_edit_weight_ = 0.0;
|
||||
double ewma_vector_cost_ = 0.0;
|
||||
double ewma_active_tiered_cost_ = 0.0;
|
||||
double ewma_edit_fraction_ = 0.0;
|
||||
std::size_t operations_since_transition_ = 0;
|
||||
std::size_t operations_since_shape_transition_ = 0;
|
||||
std::size_t last_edit_position_ = 0;
|
||||
std::size_t last_observed_size_ = 0;
|
||||
std::size_t last_element_bytes_ = sizeof(std::uint32_t);
|
||||
std::size_t confirmation_count_ = 0;
|
||||
std::optional<AdaptationDecision> pending_decision_;
|
||||
bool has_last_edit_ = false;
|
||||
bool initialized_ = false;
|
||||
bool decision_ready_ = false;
|
||||
std::size_t closed_windows_since_decision_ = 0;
|
||||
};
|
||||
|
||||
} // namespace uc
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,177 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace uc::detail {
|
||||
|
||||
// Flat open-addressed value -> duplicate-chain metadata. The actual duplicate
|
||||
// links live in AdaptiveSequence's dense ID table, so one bucket is paid per
|
||||
// distinct value rather than one node allocation per element.
|
||||
template <class Key, class Hash = std::hash<Key>, class Equal = std::equal_to<Key>>
|
||||
class FlatDuplicateIndex {
|
||||
public:
|
||||
using id_type = std::uint32_t;
|
||||
static constexpr id_type invalid_id = std::numeric_limits<id_type>::max();
|
||||
|
||||
struct Entry {
|
||||
id_type head = invalid_id;
|
||||
std::uint32_t count = 0;
|
||||
};
|
||||
|
||||
FlatDuplicateIndex() { rehash(16); }
|
||||
|
||||
[[nodiscard]] const Entry* find(const Key& key) const {
|
||||
const auto position = find_existing(key);
|
||||
return position == npos ? nullptr : &buckets_[position].entry;
|
||||
}
|
||||
|
||||
[[nodiscard]] Entry* find(const Key& key) {
|
||||
const auto position = find_existing(key);
|
||||
return position == npos ? nullptr : &buckets_[position].entry;
|
||||
}
|
||||
|
||||
Entry& ensure(const Key& key) {
|
||||
maybe_grow();
|
||||
const auto [position, found] = find_insert_position(key);
|
||||
auto& bucket = buckets_[position];
|
||||
if (!found) {
|
||||
if (bucket.state == State::tombstone) {
|
||||
--tombstones_;
|
||||
}
|
||||
bucket.key.emplace(key);
|
||||
bucket.entry = {};
|
||||
bucket.state = State::occupied;
|
||||
++size_;
|
||||
}
|
||||
return bucket.entry;
|
||||
}
|
||||
|
||||
bool erase_key(const Key& key) {
|
||||
const auto position = find_existing(key);
|
||||
if (position == npos) {
|
||||
return false;
|
||||
}
|
||||
auto& bucket = buckets_[position];
|
||||
bucket.key.reset();
|
||||
bucket.entry = {};
|
||||
bucket.state = State::tombstone;
|
||||
--size_;
|
||||
++tombstones_;
|
||||
if (tombstones_ > buckets_.size() / 4) {
|
||||
rehash(buckets_.size());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
buckets_.clear();
|
||||
size_ = 0;
|
||||
tombstones_ = 0;
|
||||
rehash(16);
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t distinct_values() const noexcept { return size_; }
|
||||
[[nodiscard]] std::size_t bucket_count() const noexcept { return buckets_.size(); }
|
||||
[[nodiscard]] std::size_t allocated_bytes() const noexcept {
|
||||
return buckets_.capacity() * sizeof(Bucket);
|
||||
}
|
||||
|
||||
private:
|
||||
enum class State : std::uint8_t { empty, occupied, tombstone };
|
||||
|
||||
struct Bucket {
|
||||
std::optional<Key> key;
|
||||
Entry entry;
|
||||
State state = State::empty;
|
||||
};
|
||||
|
||||
static constexpr std::size_t npos = std::numeric_limits<std::size_t>::max();
|
||||
|
||||
[[nodiscard]] std::size_t mask() const noexcept { return buckets_.size() - 1; }
|
||||
|
||||
[[nodiscard]] std::size_t find_existing(const Key& key) const {
|
||||
if (buckets_.empty()) {
|
||||
return npos;
|
||||
}
|
||||
auto position = hasher_(key) & mask();
|
||||
for (std::size_t probe = 0; probe < buckets_.size(); ++probe) {
|
||||
const auto& bucket = buckets_[position];
|
||||
if (bucket.state == State::empty) {
|
||||
return npos;
|
||||
}
|
||||
if (bucket.state == State::occupied && equal_(*bucket.key, key)) {
|
||||
return position;
|
||||
}
|
||||
position = (position + 1) & mask();
|
||||
}
|
||||
return npos;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::pair<std::size_t, bool>
|
||||
find_insert_position(const Key& key) const {
|
||||
auto position = hasher_(key) & mask();
|
||||
auto first_tombstone = npos;
|
||||
for (std::size_t probe = 0; probe < buckets_.size(); ++probe) {
|
||||
const auto& bucket = buckets_[position];
|
||||
if (bucket.state == State::empty) {
|
||||
return {first_tombstone == npos ? position : first_tombstone, false};
|
||||
}
|
||||
if (bucket.state == State::tombstone) {
|
||||
if (first_tombstone == npos) {
|
||||
first_tombstone = position;
|
||||
}
|
||||
} else if (equal_(*bucket.key, key)) {
|
||||
return {position, true};
|
||||
}
|
||||
position = (position + 1) & mask();
|
||||
}
|
||||
if (first_tombstone != npos) {
|
||||
return {first_tombstone, false};
|
||||
}
|
||||
throw std::length_error("flat hash index is full");
|
||||
}
|
||||
|
||||
void maybe_grow() {
|
||||
if ((size_ + tombstones_ + 1) * 100 >= buckets_.size() * 82) {
|
||||
rehash(buckets_.size() * 2);
|
||||
}
|
||||
}
|
||||
|
||||
void rehash(std::size_t requested_capacity) {
|
||||
std::size_t capacity = 16;
|
||||
while (capacity < requested_capacity) {
|
||||
capacity *= 2;
|
||||
}
|
||||
auto old = std::move(buckets_);
|
||||
buckets_.assign(capacity, Bucket{});
|
||||
size_ = 0;
|
||||
tombstones_ = 0;
|
||||
for (auto& bucket : old) {
|
||||
if (bucket.state != State::occupied) {
|
||||
continue;
|
||||
}
|
||||
const auto [position, found] = find_insert_position(*bucket.key);
|
||||
(void)found;
|
||||
auto& target = buckets_[position];
|
||||
target.key.emplace(std::move(*bucket.key));
|
||||
target.entry = bucket.entry;
|
||||
target.state = State::occupied;
|
||||
++size_;
|
||||
}
|
||||
}
|
||||
|
||||
Hash hasher_{};
|
||||
Equal equal_{};
|
||||
std::vector<Bucket> buckets_;
|
||||
std::size_t size_ = 0;
|
||||
std::size_t tombstones_ = 0;
|
||||
};
|
||||
|
||||
} // namespace uc::detail
|
||||
@@ -0,0 +1,206 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace uc::detail {
|
||||
|
||||
// A compact directory over leaf sizes. Level zero describes leaves, every
|
||||
// next level groups `fanout` children, and the requested maximum number of
|
||||
// levels is respected. The top level uses binary search; descent only scans a
|
||||
// single fanout-sized group. A Fenwick side index supplies O(log L) prefix
|
||||
// sums needed by stable-ID -> logical-index resolution.
|
||||
class HierarchicalSizeIndex {
|
||||
public:
|
||||
struct Location {
|
||||
std::size_t leaf = 0;
|
||||
std::size_t local = 0;
|
||||
};
|
||||
|
||||
HierarchicalSizeIndex(std::size_t fanout = 16, std::size_t max_levels = 3)
|
||||
: fanout_(std::max<std::size_t>(2, fanout)),
|
||||
max_levels_(std::clamp<std::size_t>(max_levels, 1, 8)) {}
|
||||
|
||||
void configure(std::size_t fanout, std::size_t max_levels) {
|
||||
const auto leaf_sizes = levels_.empty()
|
||||
? std::vector<std::size_t>{} : levels_.front();
|
||||
HierarchicalSizeIndex replacement(fanout, max_levels);
|
||||
replacement.rebuild(leaf_sizes);
|
||||
swap(replacement);
|
||||
}
|
||||
|
||||
void rebuild(const std::vector<std::size_t>& leaf_sizes) {
|
||||
// Build all allocation-owning state off to the side. A failed
|
||||
// allocation must not leave a directory that no longer describes its
|
||||
// storage.
|
||||
std::vector<std::vector<std::size_t>> new_levels;
|
||||
std::vector<std::size_t> new_top_prefix;
|
||||
std::vector<std::size_t> new_fenwick(leaf_sizes.size() + 1, 0);
|
||||
std::size_t new_total = 0;
|
||||
if (leaf_sizes.empty()) {
|
||||
levels_.swap(new_levels);
|
||||
top_prefix_.swap(new_top_prefix);
|
||||
fenwick_.swap(new_fenwick);
|
||||
total_ = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
new_levels.push_back(leaf_sizes);
|
||||
// Keep the top directory at most fanout-sized. Binary search handles
|
||||
// that top level, so building extra single-parent levels only adds a
|
||||
// lookup and cannot improve asymptotic complexity.
|
||||
while (new_levels.back().size() > fanout_
|
||||
&& new_levels.size() < max_levels_) {
|
||||
const auto& children = new_levels.back();
|
||||
std::vector<std::size_t> parents;
|
||||
parents.reserve((children.size() + fanout_ - 1) / fanout_);
|
||||
for (std::size_t begin = 0; begin < children.size(); begin += fanout_) {
|
||||
const auto end = std::min(children.size(), begin + fanout_);
|
||||
std::size_t weight = 0;
|
||||
for (auto i = begin; i < end; ++i) {
|
||||
weight += children[i];
|
||||
}
|
||||
parents.push_back(weight);
|
||||
}
|
||||
new_levels.push_back(std::move(parents));
|
||||
}
|
||||
|
||||
for (const auto weight : new_levels.back()) {
|
||||
new_total += weight;
|
||||
new_top_prefix.push_back(new_total);
|
||||
}
|
||||
for (std::size_t i = 0; i < leaf_sizes.size(); ++i) {
|
||||
for (auto node = i + 1; node < new_fenwick.size();
|
||||
node += node & (~node + 1)) {
|
||||
new_fenwick[node] += leaf_sizes[i];
|
||||
}
|
||||
}
|
||||
|
||||
levels_.swap(new_levels);
|
||||
top_prefix_.swap(new_top_prefix);
|
||||
fenwick_.swap(new_fenwick);
|
||||
total_ = new_total;
|
||||
}
|
||||
|
||||
void swap(HierarchicalSizeIndex& other) noexcept {
|
||||
using std::swap;
|
||||
swap(fanout_, other.fanout_);
|
||||
swap(max_levels_, other.max_levels_);
|
||||
swap(total_, other.total_);
|
||||
levels_.swap(other.levels_);
|
||||
top_prefix_.swap(other.top_prefix_);
|
||||
fenwick_.swap(other.fenwick_);
|
||||
}
|
||||
|
||||
void update(std::size_t leaf, std::ptrdiff_t delta) {
|
||||
if (levels_.empty() || leaf >= levels_.front().size()) {
|
||||
throw std::out_of_range("directory leaf index out of range");
|
||||
}
|
||||
apply_delta(levels_[0][leaf], delta);
|
||||
std::size_t node = leaf;
|
||||
for (std::size_t level = 1; level < levels_.size(); ++level) {
|
||||
node /= fanout_;
|
||||
apply_delta(levels_[level][node], delta);
|
||||
}
|
||||
total_ = apply_delta_copy(total_, delta);
|
||||
fenwick_add_signed(leaf, delta);
|
||||
rebuild_top_prefix();
|
||||
}
|
||||
|
||||
[[nodiscard]] Location locate(std::size_t logical_index) const {
|
||||
if (logical_index >= total_ || levels_.empty()) {
|
||||
throw std::out_of_range("logical index out of range");
|
||||
}
|
||||
|
||||
const auto top_it = std::upper_bound(top_prefix_.begin(), top_prefix_.end(), logical_index);
|
||||
std::size_t node = static_cast<std::size_t>(top_it - top_prefix_.begin());
|
||||
std::size_t remaining = logical_index - (node == 0 ? 0 : top_prefix_[node - 1]);
|
||||
|
||||
for (std::size_t level = levels_.size() - 1; level > 0; --level) {
|
||||
const auto& children = levels_[level - 1];
|
||||
const auto begin = node * fanout_;
|
||||
const auto end = std::min(children.size(), begin + fanout_);
|
||||
auto child = begin;
|
||||
for (; child < end; ++child) {
|
||||
if (remaining < children[child]) {
|
||||
break;
|
||||
}
|
||||
remaining -= children[child];
|
||||
}
|
||||
assert(child < end);
|
||||
node = child;
|
||||
}
|
||||
return {node, remaining};
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t prefix_before(std::size_t leaf) const noexcept {
|
||||
std::size_t sum = 0;
|
||||
for (std::size_t i = leaf; i > 0; i -= i & (~i + 1)) {
|
||||
sum += fenwick_[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t total() const noexcept { return total_; }
|
||||
[[nodiscard]] std::size_t actual_levels() const noexcept { return levels_.size(); }
|
||||
[[nodiscard]] std::size_t fanout() const noexcept { return fanout_; }
|
||||
|
||||
[[nodiscard]] std::size_t allocated_bytes() const noexcept {
|
||||
std::size_t bytes = fenwick_.capacity() * sizeof(std::size_t)
|
||||
+ top_prefix_.capacity() * sizeof(std::size_t);
|
||||
for (const auto& level : levels_) {
|
||||
bytes += level.capacity() * sizeof(std::size_t);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private:
|
||||
static void apply_delta(std::size_t& value, std::ptrdiff_t delta) {
|
||||
value = apply_delta_copy(value, delta);
|
||||
}
|
||||
|
||||
static std::size_t apply_delta_copy(std::size_t value, std::ptrdiff_t delta) {
|
||||
if (delta < 0) {
|
||||
const auto magnitude = static_cast<std::size_t>(-delta);
|
||||
assert(value >= magnitude);
|
||||
return value - magnitude;
|
||||
}
|
||||
return value + static_cast<std::size_t>(delta);
|
||||
}
|
||||
|
||||
void rebuild_top_prefix() {
|
||||
top_prefix_.clear();
|
||||
top_prefix_.reserve(levels_.back().size());
|
||||
std::size_t prefix = 0;
|
||||
for (const auto weight : levels_.back()) {
|
||||
prefix += weight;
|
||||
top_prefix_.push_back(prefix);
|
||||
}
|
||||
}
|
||||
|
||||
void fenwick_add(std::size_t leaf, std::size_t delta) noexcept {
|
||||
for (auto i = leaf + 1; i < fenwick_.size(); i += i & (~i + 1)) {
|
||||
fenwick_[i] += delta;
|
||||
}
|
||||
}
|
||||
|
||||
void fenwick_add_signed(std::size_t leaf, std::ptrdiff_t delta) noexcept {
|
||||
for (auto i = leaf + 1; i < fenwick_.size(); i += i & (~i + 1)) {
|
||||
fenwick_[i] = apply_delta_copy(fenwick_[i], delta);
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t fanout_;
|
||||
std::size_t max_levels_;
|
||||
std::size_t total_ = 0;
|
||||
std::vector<std::vector<std::size_t>> levels_;
|
||||
std::vector<std::size_t> top_prefix_;
|
||||
std::vector<std::size_t> fenwick_;
|
||||
};
|
||||
|
||||
} // namespace uc::detail
|
||||
@@ -0,0 +1,184 @@
|
||||
#pragma once
|
||||
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace uc::detail {
|
||||
|
||||
// A fixed-capacity circular leaf. Empty slots are represented explicitly so
|
||||
// the block also works for non-default-constructible and non-trivial values.
|
||||
// Insert/erase shifts the shorter side and therefore moves at most size()/2
|
||||
// elements. offset_ is the physical slot containing logical element zero.
|
||||
template <class T>
|
||||
class RingBlock {
|
||||
public:
|
||||
explicit RingBlock(std::size_t capacity = 256)
|
||||
: slots_(capacity) {
|
||||
if (capacity < 2) {
|
||||
throw std::invalid_argument("RingBlock capacity must be at least two");
|
||||
}
|
||||
}
|
||||
|
||||
RingBlock(const RingBlock&) = default;
|
||||
RingBlock(RingBlock&&) noexcept = default;
|
||||
RingBlock& operator=(const RingBlock&) = default;
|
||||
RingBlock& operator=(RingBlock&&) noexcept = default;
|
||||
~RingBlock() = default;
|
||||
|
||||
[[nodiscard]] std::size_t size() const noexcept { return size_; }
|
||||
[[nodiscard]] std::size_t capacity() const noexcept { return slots_.size(); }
|
||||
[[nodiscard]] bool empty() const noexcept { return size_ == 0; }
|
||||
[[nodiscard]] bool full() const noexcept { return size_ == capacity(); }
|
||||
[[nodiscard]] std::size_t offset() const noexcept { return offset_; }
|
||||
|
||||
T& operator[](std::size_t index) noexcept {
|
||||
assert(index < size_);
|
||||
return *slots_[physical(index)];
|
||||
}
|
||||
|
||||
const T& operator[](std::size_t index) const noexcept {
|
||||
assert(index < size_);
|
||||
return *slots_[physical(index)];
|
||||
}
|
||||
|
||||
T& at(std::size_t index) {
|
||||
if (index >= size_) {
|
||||
throw std::out_of_range("RingBlock index out of range");
|
||||
}
|
||||
return (*this)[index];
|
||||
}
|
||||
|
||||
const T& at(std::size_t index) const {
|
||||
if (index >= size_) {
|
||||
throw std::out_of_range("RingBlock index out of range");
|
||||
}
|
||||
return (*this)[index];
|
||||
}
|
||||
|
||||
template <class U>
|
||||
void insert(std::size_t index, U&& value) {
|
||||
if (index > size_) {
|
||||
throw std::out_of_range("RingBlock insertion index out of range");
|
||||
}
|
||||
if (full()) {
|
||||
throw std::length_error("RingBlock is full");
|
||||
}
|
||||
|
||||
const auto old_offset = offset_;
|
||||
if (index < size_ / 2) {
|
||||
const auto new_offset = decrement(old_offset);
|
||||
for (std::size_t i = 0; i < index; ++i) {
|
||||
move_slot((new_offset + i) % capacity(), (old_offset + i) % capacity());
|
||||
}
|
||||
offset_ = new_offset;
|
||||
} else {
|
||||
for (std::size_t i = size_; i > index; --i) {
|
||||
move_slot((old_offset + i) % capacity(),
|
||||
(old_offset + i - 1) % capacity());
|
||||
}
|
||||
}
|
||||
|
||||
slots_[physical(index)].emplace(std::forward<U>(value));
|
||||
++size_;
|
||||
assert_invariant();
|
||||
}
|
||||
|
||||
template <class... Args>
|
||||
T& emplace(std::size_t index, Args&&... args) {
|
||||
T value(std::forward<Args>(args)...);
|
||||
insert(index, std::move(value));
|
||||
return (*this)[index];
|
||||
}
|
||||
|
||||
template <class U>
|
||||
void push_back(U&& value) {
|
||||
insert(size_, std::forward<U>(value));
|
||||
}
|
||||
|
||||
T erase(std::size_t index) {
|
||||
if (index >= size_) {
|
||||
throw std::out_of_range("RingBlock erase index out of range");
|
||||
}
|
||||
|
||||
T removed(std::move((*this)[index]));
|
||||
const auto old_offset = offset_;
|
||||
slots_[physical(index)].reset();
|
||||
|
||||
if (index < size_ / 2) {
|
||||
for (std::size_t i = index; i > 0; --i) {
|
||||
move_slot((old_offset + i) % capacity(),
|
||||
(old_offset + i - 1) % capacity());
|
||||
}
|
||||
offset_ = increment(old_offset);
|
||||
} else {
|
||||
for (std::size_t i = index; i + 1 < size_; ++i) {
|
||||
move_slot((old_offset + i) % capacity(),
|
||||
(old_offset + i + 1) % capacity());
|
||||
}
|
||||
}
|
||||
|
||||
--size_;
|
||||
if (size_ == 0) {
|
||||
offset_ = 0;
|
||||
}
|
||||
assert_invariant();
|
||||
return removed;
|
||||
}
|
||||
|
||||
void clear() noexcept {
|
||||
for (auto& slot : slots_) {
|
||||
slot.reset();
|
||||
}
|
||||
size_ = 0;
|
||||
offset_ = 0;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t allocated_bytes() const noexcept {
|
||||
return slots_.capacity() * sizeof(typename decltype(slots_)::value_type);
|
||||
}
|
||||
|
||||
private:
|
||||
[[nodiscard]] std::size_t physical(std::size_t logical) const noexcept {
|
||||
return (offset_ + logical) % capacity();
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t increment(std::size_t slot) const noexcept {
|
||||
return slot + 1 == capacity() ? 0 : slot + 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t decrement(std::size_t slot) const noexcept {
|
||||
return slot == 0 ? capacity() - 1 : slot - 1;
|
||||
}
|
||||
|
||||
void move_slot(std::size_t destination, std::size_t source) {
|
||||
assert(!slots_[destination].has_value());
|
||||
assert(slots_[source].has_value());
|
||||
slots_[destination].emplace(std::move(*slots_[source]));
|
||||
slots_[source].reset();
|
||||
}
|
||||
|
||||
void assert_invariant() const noexcept {
|
||||
#ifndef NDEBUG
|
||||
std::size_t occupied = 0;
|
||||
for (const auto& slot : slots_) {
|
||||
occupied += slot.has_value() ? 1U : 0U;
|
||||
}
|
||||
assert(occupied == size_);
|
||||
for (std::size_t i = 0; i < size_; ++i) {
|
||||
assert(slots_[physical(i)].has_value());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
std::vector<std::optional<T>> slots_;
|
||||
std::size_t offset_ = 0;
|
||||
std::size_t size_ = 0;
|
||||
};
|
||||
|
||||
} // namespace uc::detail
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
#pragma once
|
||||
|
||||
#include "universal_container/hierarchical_size_index.hpp"
|
||||
#include "universal_container/ring_block.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace uc {
|
||||
|
||||
struct TieredConfig {
|
||||
std::size_t leaf_capacity = 512;
|
||||
std::size_t directory_fanout = 64;
|
||||
std::size_t directory_levels = 4;
|
||||
|
||||
friend bool operator==(const TieredConfig&, const TieredConfig&) = default;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
|
||||
template <class T>
|
||||
class TieredStorage {
|
||||
public:
|
||||
using value_type = T;
|
||||
using leaf_id_type = std::uint32_t;
|
||||
static constexpr std::size_t npos = std::numeric_limits<std::size_t>::max();
|
||||
|
||||
explicit TieredStorage(TieredConfig config = {})
|
||||
: config_(normalize(config)),
|
||||
directory_(config_.directory_fanout, config_.directory_levels) {}
|
||||
|
||||
TieredStorage(const TieredStorage& other)
|
||||
: config_(other.config_),
|
||||
directory_(config_.directory_fanout, config_.directory_levels),
|
||||
size_(other.size_),
|
||||
next_leaf_id_(other.next_leaf_id_),
|
||||
leaf_positions_(other.leaf_positions_) {
|
||||
leaves_.reserve(other.leaves_.size());
|
||||
for (const auto& leaf : other.leaves_) {
|
||||
leaves_.push_back(std::make_unique<Leaf>(*leaf));
|
||||
}
|
||||
rebuild_directory();
|
||||
}
|
||||
|
||||
TieredStorage(TieredStorage&&) noexcept = default;
|
||||
|
||||
TieredStorage& operator=(const TieredStorage& other) {
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
TieredStorage copy(other);
|
||||
swap(copy);
|
||||
return *this;
|
||||
}
|
||||
|
||||
TieredStorage& operator=(TieredStorage&&) noexcept = default;
|
||||
~TieredStorage() = default;
|
||||
|
||||
void swap(TieredStorage& other) noexcept {
|
||||
using std::swap;
|
||||
swap(config_, other.config_);
|
||||
swap(leaves_, other.leaves_);
|
||||
swap(directory_, other.directory_);
|
||||
swap(size_, other.size_);
|
||||
swap(next_leaf_id_, other.next_leaf_id_);
|
||||
swap(leaf_positions_, other.leaf_positions_);
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t size() const noexcept { return size_; }
|
||||
[[nodiscard]] bool empty() const noexcept { return size_ == 0; }
|
||||
[[nodiscard]] const TieredConfig& config() const noexcept { return config_; }
|
||||
[[nodiscard]] std::size_t leaf_count() const noexcept { return leaves_.size(); }
|
||||
[[nodiscard]] std::size_t actual_levels() const noexcept { return directory_.actual_levels(); }
|
||||
|
||||
// Changing only the directory geometry does not move elements or leaves.
|
||||
// It is substantially cheaper than rebuilding with another leaf capacity.
|
||||
void reconfigure_directory(std::size_t fanout, std::size_t maximum_levels) {
|
||||
auto requested = config_;
|
||||
requested.directory_fanout = fanout;
|
||||
requested.directory_levels = maximum_levels;
|
||||
requested = normalize(requested);
|
||||
if (requested.directory_fanout == config_.directory_fanout
|
||||
&& requested.directory_levels == config_.directory_levels) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Build the complete replacement before changing either the active
|
||||
// directory or the public configuration. HierarchicalSizeIndex::rebuild
|
||||
// allocates several vectors and may throw; mutating directory_ first
|
||||
// would otherwise leave locate() unusable after an allocation failure.
|
||||
std::vector<std::size_t> sizes;
|
||||
sizes.reserve(leaves_.size());
|
||||
for (const auto& leaf : leaves_) {
|
||||
sizes.push_back(leaf->values.size());
|
||||
}
|
||||
HierarchicalSizeIndex rebuilt(requested.directory_fanout,
|
||||
requested.directory_levels);
|
||||
rebuilt.rebuild(sizes);
|
||||
|
||||
static_assert(std::is_nothrow_move_assignable_v<HierarchicalSizeIndex>);
|
||||
directory_ = std::move(rebuilt);
|
||||
config_.directory_fanout = requested.directory_fanout;
|
||||
config_.directory_levels = requested.directory_levels;
|
||||
}
|
||||
|
||||
T& operator[](std::size_t index) noexcept {
|
||||
const auto location = locate_unchecked(index);
|
||||
return leaves_[location.leaf]->values[location.local];
|
||||
}
|
||||
|
||||
const T& operator[](std::size_t index) const noexcept {
|
||||
const auto location = locate_unchecked(index);
|
||||
return leaves_[location.leaf]->values[location.local];
|
||||
}
|
||||
|
||||
T& at(std::size_t index) {
|
||||
check_index(index);
|
||||
return (*this)[index];
|
||||
}
|
||||
|
||||
const T& at(std::size_t index) const {
|
||||
check_index(index);
|
||||
return (*this)[index];
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
void push_back(T value, Relocate&& relocate) {
|
||||
if (leaves_.empty() || leaves_.back()->values.full()) {
|
||||
leaves_.push_back(make_leaf());
|
||||
rebuild_positions();
|
||||
rebuild_directory();
|
||||
}
|
||||
auto& leaf = *leaves_.back();
|
||||
leaf.values.push_back(std::move(value));
|
||||
++size_;
|
||||
directory_.update(leaves_.size() - 1, 1);
|
||||
relocate(leaf.values[leaf.values.size() - 1], leaf.id, leaf.values.size() - 1);
|
||||
}
|
||||
|
||||
void push_back(T value) {
|
||||
push_back(std::move(value), NoRelocate{});
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
void insert(std::size_t index, T value, Relocate&& relocate) {
|
||||
if (index > size_) {
|
||||
throw std::out_of_range("tiered insertion index out of range");
|
||||
}
|
||||
if (index == size_) {
|
||||
push_back(std::move(value), std::forward<Relocate>(relocate));
|
||||
return;
|
||||
}
|
||||
|
||||
const auto location = directory_.locate(index);
|
||||
auto& leaf = *leaves_[location.leaf];
|
||||
if (!leaf.values.full()) {
|
||||
leaf.values.insert(location.local, std::move(value));
|
||||
++size_;
|
||||
directory_.update(location.leaf, 1);
|
||||
refresh_leaf(location.leaf, relocate);
|
||||
return;
|
||||
}
|
||||
|
||||
split_and_insert(location.leaf, location.local, std::move(value), relocate);
|
||||
++size_;
|
||||
rebuild_directory();
|
||||
}
|
||||
|
||||
void insert(std::size_t index, T value) {
|
||||
insert(index, std::move(value), NoRelocate{});
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
T erase(std::size_t index, Relocate&& relocate) {
|
||||
check_index(index);
|
||||
const auto location = directory_.locate(index);
|
||||
auto removed = leaves_[location.leaf]->values.erase(location.local);
|
||||
--size_;
|
||||
|
||||
if (leaves_[location.leaf]->values.empty() && leaves_.size() > 1) {
|
||||
retire_leaf(location.leaf);
|
||||
rebuild_positions();
|
||||
rebuild_directory();
|
||||
return removed;
|
||||
}
|
||||
|
||||
if (try_merge(location.leaf, relocate)) {
|
||||
return removed;
|
||||
}
|
||||
|
||||
directory_.update(location.leaf, -1);
|
||||
refresh_leaf(location.leaf, relocate);
|
||||
return removed;
|
||||
}
|
||||
|
||||
T erase(std::size_t index) {
|
||||
return erase(index, NoRelocate{});
|
||||
}
|
||||
|
||||
void clear() noexcept {
|
||||
leaves_.clear();
|
||||
leaf_positions_.clear();
|
||||
size_ = 0;
|
||||
next_leaf_id_ = 0;
|
||||
directory_.rebuild({});
|
||||
}
|
||||
|
||||
template <class Function>
|
||||
void for_each(Function&& function) {
|
||||
for (auto& leaf : leaves_) {
|
||||
for (std::size_t i = 0; i < leaf->values.size(); ++i) {
|
||||
function(leaf->values[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class Function>
|
||||
void for_each(Function&& function) const {
|
||||
for (const auto& leaf : leaves_) {
|
||||
for (std::size_t i = 0; i < leaf->values.size(); ++i) {
|
||||
function(leaf->values[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class Function>
|
||||
void for_each_with_location(Function&& function) const {
|
||||
for (const auto& leaf : leaves_) {
|
||||
for (std::size_t i = 0; i < leaf->values.size(); ++i) {
|
||||
function(leaf->values[i], leaf->id, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] std::vector<T> to_vector_copy() const {
|
||||
std::vector<T> result;
|
||||
result.reserve(size_);
|
||||
for_each([&](const T& value) { result.push_back(value); });
|
||||
return result;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::vector<T> to_vector_move() {
|
||||
std::vector<T> result;
|
||||
result.reserve(size_);
|
||||
for_each([&](T& value) { result.push_back(std::move(value)); });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
static TieredStorage from_vector(std::vector<T>&& source,
|
||||
TieredConfig config,
|
||||
Relocate&& relocate) {
|
||||
TieredStorage result(config);
|
||||
if (source.empty()) {
|
||||
return result;
|
||||
}
|
||||
const auto target_occupancy = bulk_target_occupancy(
|
||||
result.config_.leaf_capacity);
|
||||
const auto count = (source.size() + target_occupancy - 1)
|
||||
/ target_occupancy;
|
||||
result.leaves_.reserve(count);
|
||||
for (auto& value : source) {
|
||||
if (result.leaves_.empty()
|
||||
|| result.leaves_.back()->values.size() == target_occupancy) {
|
||||
result.leaves_.push_back(result.make_leaf());
|
||||
}
|
||||
result.leaves_.back()->values.push_back(std::move(value));
|
||||
++result.size_;
|
||||
}
|
||||
result.rebuild_positions();
|
||||
result.rebuild_directory();
|
||||
for (std::size_t leaf = 0; leaf < result.leaves_.size(); ++leaf) {
|
||||
result.refresh_leaf(leaf, relocate);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static TieredStorage from_vector(std::vector<T>&& source, TieredConfig config) {
|
||||
return from_vector(std::move(source), config, NoRelocate{});
|
||||
}
|
||||
|
||||
static TieredStorage from_vector_copy(const std::vector<T>& source,
|
||||
TieredConfig config) {
|
||||
TieredStorage result(config);
|
||||
if (source.empty()) {
|
||||
return result;
|
||||
}
|
||||
const auto target_occupancy = bulk_target_occupancy(
|
||||
result.config_.leaf_capacity);
|
||||
const auto count = (source.size() + target_occupancy - 1)
|
||||
/ target_occupancy;
|
||||
result.leaves_.reserve(count);
|
||||
for (const auto& value : source) {
|
||||
if (result.leaves_.empty()
|
||||
|| result.leaves_.back()->values.size() == target_occupancy) {
|
||||
result.leaves_.push_back(result.make_leaf());
|
||||
}
|
||||
result.leaves_.back()->values.push_back(value);
|
||||
++result.size_;
|
||||
}
|
||||
result.rebuild_positions();
|
||||
result.rebuild_directory();
|
||||
return result;
|
||||
}
|
||||
|
||||
static TieredStorage reconfigured_copy(const TieredStorage& source,
|
||||
TieredConfig config) {
|
||||
TieredStorage result(config);
|
||||
if (source.empty()) {
|
||||
return result;
|
||||
}
|
||||
const auto target_occupancy = bulk_target_occupancy(
|
||||
result.config_.leaf_capacity);
|
||||
const auto count = (source.size() + target_occupancy - 1)
|
||||
/ target_occupancy;
|
||||
result.leaves_.reserve(count);
|
||||
source.for_each([&](const T& value) {
|
||||
if (result.leaves_.empty()
|
||||
|| result.leaves_.back()->values.size() == target_occupancy) {
|
||||
result.leaves_.push_back(result.make_leaf());
|
||||
}
|
||||
result.leaves_.back()->values.push_back(value);
|
||||
++result.size_;
|
||||
});
|
||||
result.rebuild_positions();
|
||||
result.rebuild_directory();
|
||||
return result;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t logical_index(leaf_id_type leaf_id,
|
||||
std::size_t local) const {
|
||||
if (leaf_id >= leaf_positions_.size()) {
|
||||
throw std::out_of_range("unknown tiered leaf id");
|
||||
}
|
||||
const auto position = leaf_positions_[leaf_id];
|
||||
if (position == npos || local >= leaves_[position]->values.size()) {
|
||||
throw std::out_of_range("stale tiered location");
|
||||
}
|
||||
return directory_.prefix_before(position) + local;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t allocated_bytes() const noexcept {
|
||||
std::size_t bytes = leaves_.capacity() * sizeof(typename decltype(leaves_)::value_type)
|
||||
+ leaf_positions_.capacity() * sizeof(std::size_t)
|
||||
+ directory_.allocated_bytes();
|
||||
for (const auto& leaf : leaves_) {
|
||||
bytes += sizeof(Leaf) + leaf->values.allocated_bytes();
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private:
|
||||
struct NoRelocate {
|
||||
void operator()(const T&, leaf_id_type, std::size_t) const noexcept {}
|
||||
};
|
||||
|
||||
struct Leaf {
|
||||
Leaf(leaf_id_type leaf_id, std::size_t capacity)
|
||||
: id(leaf_id), values(capacity) {}
|
||||
|
||||
leaf_id_type id;
|
||||
RingBlock<T> values;
|
||||
};
|
||||
|
||||
static TieredConfig normalize(TieredConfig config) {
|
||||
config.leaf_capacity = std::clamp<std::size_t>(config.leaf_capacity, 4, 1U << 20U);
|
||||
config.directory_fanout = std::clamp<std::size_t>(config.directory_fanout, 2, 1U << 16U);
|
||||
config.directory_levels = std::clamp<std::size_t>(config.directory_levels, 1, 8);
|
||||
return config;
|
||||
}
|
||||
|
||||
// Bulk-loading completely full leaves makes the first random insertion
|
||||
// into almost every leaf pay for a split and a directory rebuild. Keep a
|
||||
// small, deterministic reserve instead. Seven eighths preserves compact
|
||||
// memory usage while giving each freshly built leaf enough room for the
|
||||
// short edit bursts that motivate switching away from vector storage.
|
||||
[[nodiscard]] static std::size_t
|
||||
bulk_target_occupancy(std::size_t capacity) noexcept {
|
||||
return std::max<std::size_t>(1, capacity - capacity / 8);
|
||||
}
|
||||
|
||||
[[nodiscard]] std::unique_ptr<Leaf> make_leaf() {
|
||||
if (next_leaf_id_ == std::numeric_limits<leaf_id_type>::max()) {
|
||||
throw std::length_error("tiered leaf id space exhausted");
|
||||
}
|
||||
const auto id = next_leaf_id_++;
|
||||
if (leaf_positions_.size() <= id) {
|
||||
leaf_positions_.resize(static_cast<std::size_t>(id) + 1, npos);
|
||||
}
|
||||
return std::make_unique<Leaf>(id, config_.leaf_capacity);
|
||||
}
|
||||
|
||||
void check_index(std::size_t index) const {
|
||||
if (index >= size_) {
|
||||
throw std::out_of_range("tiered index out of range");
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] typename HierarchicalSizeIndex::Location
|
||||
locate_unchecked(std::size_t index) const noexcept {
|
||||
assert(index < size_);
|
||||
try {
|
||||
return directory_.locate(index);
|
||||
} catch (...) {
|
||||
std::terminate();
|
||||
}
|
||||
}
|
||||
|
||||
void rebuild_positions() {
|
||||
std::fill(leaf_positions_.begin(), leaf_positions_.end(), npos);
|
||||
for (std::size_t i = 0; i < leaves_.size(); ++i) {
|
||||
const auto id = leaves_[i]->id;
|
||||
if (leaf_positions_.size() <= id) {
|
||||
leaf_positions_.resize(static_cast<std::size_t>(id) + 1, npos);
|
||||
}
|
||||
leaf_positions_[id] = i;
|
||||
}
|
||||
}
|
||||
|
||||
void rebuild_directory() {
|
||||
std::vector<std::size_t> sizes;
|
||||
sizes.reserve(leaves_.size());
|
||||
for (const auto& leaf : leaves_) {
|
||||
sizes.push_back(leaf->values.size());
|
||||
}
|
||||
directory_.rebuild(sizes);
|
||||
assert(directory_.total() == size_);
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
void refresh_leaf(std::size_t leaf_index, Relocate& relocate) {
|
||||
auto& leaf = *leaves_[leaf_index];
|
||||
for (std::size_t i = 0; i < leaf.values.size(); ++i) {
|
||||
relocate(leaf.values[i], leaf.id, i);
|
||||
}
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
void split_and_insert(std::size_t leaf_index,
|
||||
std::size_t local,
|
||||
T value,
|
||||
Relocate& relocate) {
|
||||
auto& old = *leaves_[leaf_index];
|
||||
std::vector<T> combined;
|
||||
combined.reserve(old.values.size() + 1);
|
||||
for (std::size_t i = 0; i < old.values.size(); ++i) {
|
||||
combined.push_back(std::move(old.values[i]));
|
||||
}
|
||||
combined.insert(combined.begin() + static_cast<std::ptrdiff_t>(local), std::move(value));
|
||||
|
||||
const auto left_id = old.id;
|
||||
auto left = std::make_unique<Leaf>(left_id, config_.leaf_capacity);
|
||||
auto right = make_leaf();
|
||||
const auto middle = combined.size() / 2;
|
||||
for (std::size_t i = 0; i < middle; ++i) {
|
||||
left->values.push_back(std::move(combined[i]));
|
||||
}
|
||||
for (std::size_t i = middle; i < combined.size(); ++i) {
|
||||
right->values.push_back(std::move(combined[i]));
|
||||
}
|
||||
|
||||
leaves_[leaf_index] = std::move(left);
|
||||
leaves_.insert(leaves_.begin() + static_cast<std::ptrdiff_t>(leaf_index + 1),
|
||||
std::move(right));
|
||||
rebuild_positions();
|
||||
refresh_leaf(leaf_index, relocate);
|
||||
refresh_leaf(leaf_index + 1, relocate);
|
||||
}
|
||||
|
||||
void retire_leaf(std::size_t leaf_index) {
|
||||
const auto id = leaves_[leaf_index]->id;
|
||||
if (id < leaf_positions_.size()) {
|
||||
leaf_positions_[id] = npos;
|
||||
}
|
||||
leaves_.erase(leaves_.begin() + static_cast<std::ptrdiff_t>(leaf_index));
|
||||
}
|
||||
|
||||
template <class Relocate>
|
||||
bool try_merge(std::size_t leaf_index, Relocate& relocate) {
|
||||
if (leaves_.size() < 2) {
|
||||
return false;
|
||||
}
|
||||
const auto threshold = std::max<std::size_t>(1, config_.leaf_capacity / 4);
|
||||
if (leaves_[leaf_index]->values.size() >= threshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::size_t left_index = leaf_index;
|
||||
std::size_t right_index = leaf_index + 1;
|
||||
if (right_index >= leaves_.size()) {
|
||||
left_index = leaf_index - 1;
|
||||
right_index = leaf_index;
|
||||
}
|
||||
const auto combined_size = leaves_[left_index]->values.size()
|
||||
+ leaves_[right_index]->values.size();
|
||||
if (combined_size > config_.leaf_capacity) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto survivor_id = leaves_[left_index]->id;
|
||||
auto merged = std::make_unique<Leaf>(survivor_id, config_.leaf_capacity);
|
||||
for (std::size_t i = 0; i < leaves_[left_index]->values.size(); ++i) {
|
||||
merged->values.push_back(std::move(leaves_[left_index]->values[i]));
|
||||
}
|
||||
for (std::size_t i = 0; i < leaves_[right_index]->values.size(); ++i) {
|
||||
merged->values.push_back(std::move(leaves_[right_index]->values[i]));
|
||||
}
|
||||
|
||||
const auto retired_id = leaves_[right_index]->id;
|
||||
leaves_[left_index] = std::move(merged);
|
||||
leaves_.erase(leaves_.begin() + static_cast<std::ptrdiff_t>(right_index));
|
||||
if (retired_id < leaf_positions_.size()) {
|
||||
leaf_positions_[retired_id] = npos;
|
||||
}
|
||||
rebuild_positions();
|
||||
rebuild_directory();
|
||||
refresh_leaf(left_index, relocate);
|
||||
return true;
|
||||
}
|
||||
|
||||
TieredConfig config_;
|
||||
std::vector<std::unique_ptr<Leaf>> leaves_;
|
||||
HierarchicalSizeIndex directory_;
|
||||
std::size_t size_ = 0;
|
||||
leaf_id_type next_leaf_id_ = 0;
|
||||
std::vector<std::size_t> leaf_positions_;
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
} // namespace uc
|
||||
Reference in New Issue
Block a user