Lookich/UniversalContainer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New adaptation policy

Browse Source
This commit is contained in:
Efim Beshmenev
2026-08-13 23:42:26 +03:00
parent bbb43dbe57
commit 3770224eaf
16 changed files with 2533 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/adaptive_sequence_tests.cpp
+249 -96
View File
@@ -5,6 +5,7 @@
#include <cstdint>
#include <cstdlib>
#include <iostream>
#include <limits>
#include <optional>
#include <random>
#include <stdexcept>
@@ -72,6 +73,12 @@ struct DeferredTransitionPolicy {
}
};
struct WraparoundHash {
[[nodiscard]] std::size_t operator()(std::uint32_t) const noexcept {
return std::numeric_limits<std::size_t>::max();
}
};
template <class Sequence, class Reference>
void expect_equal(const Sequence& sequence, const Reference& reference) {
EXPECT(sequence.size() == reference.size());
@@ -172,6 +179,157 @@ void indexed_duplicates_and_ids() {
EXPECT(values.size() == 2);
}
void indexed_ids_reuse_slots_without_reviving_stale_handles() {
uc::AdaptiveSequence<std::uint32_t, true> values;
for (std::uint32_t value = 0; value < 64; ++value) {
values.push_back(value);
}
const auto first_id = values.id_at(0);
const auto fixed_allocation = values.allocated_bytes();
for (std::uint32_t step = 0; step < 2'048; ++step) {
const auto retired = values.id_at(0);
values.erase_by_id(retired);
EXPECT(!values.id_alive(retired));
values.push_back(10'000 + step);
const auto replacement = values.id_at(values.size() - 1);
EXPECT(replacement != retired);
EXPECT(values.id_alive(replacement));
EXPECT(!values.id_alive(retired));
EXPECT(!values.id_alive(first_id));
EXPECT(values.find_one(10'000 + step) == values.size() - 1);
EXPECT(values.find_all_ids(10'000 + step)
== std::vector<decltype(values)::stable_id>({replacement}));
bool stale_rejected = false;
try {
values.erase_by_id(retired);
} catch (const std::out_of_range&) {
stale_rejected = true;
}
EXPECT(stale_rejected);
}
// Churn at a fixed live size must recycle the 64 metadata slots instead
// of growing the ID vector outside a logical-capacity boundary.
EXPECT(values.size() == 64);
EXPECT(values.capacity() == 64);
EXPECT(values.allocated_bytes() == fixed_allocation);
values.set(0, 0xf00d);
values.set(1, 0xf00d);
const auto duplicate_ids = values.find_all_ids(0xf00d);
EXPECT(duplicate_ids.size() == 2);
EXPECT(values.erase_all(0xf00d) == 2);
for (const auto id : duplicate_ids) {
EXPECT(!values.id_alive(id));
}
std::vector<decltype(values)::stable_id> ids_before_clear;
ids_before_clear.reserve(values.size());
for (std::size_t index = 0; index < values.size(); ++index) {
ids_before_clear.push_back(values.id_at(index));
}
values.clear();
for (const auto id : ids_before_clear) {
EXPECT(!values.id_alive(id));
}
EXPECT(!values.id_alive(decltype(values)::invalid_id));
values.push_back(77);
const auto after_clear = values.id_at(0);
EXPECT(values.id_alive(after_clear));
EXPECT(std::find(ids_before_clear.begin(), ids_before_clear.end(), after_clear)
== ids_before_clear.end());
}
void moved_from_sequences_are_reusable_and_assignment_invalidates_proxies() {
using Sequence = uc::AdaptiveSequence<std::uint32_t, true>;
Sequence source({8, 4, 3});
for (std::uint32_t value = 0; value < 64; ++value) {
source.push_back(value);
}
source.force_tiered_mode();
const auto transferred_id = source.id_at(17);
Sequence moved(std::move(source));
EXPECT(moved.size() == 64);
EXPECT(moved.id_alive(transferred_id));
EXPECT(source.empty());
EXPECT(source.capacity() == 0);
EXPECT(source.mode() == uc::StorageMode::vector);
EXPECT(!source.id_alive(transferred_id));
for (std::uint32_t value = 0; value < 40; ++value) {
source.push_back(1'000 + value);
}
EXPECT(source.size() == 40);
EXPECT(source.contains(1'039));
Sequence destination;
destination.push_back(7);
auto stale_after_move_assignment = destination[0];
destination = std::move(moved);
bool move_assignment_invalidated = false;
try {
stale_after_move_assignment = 8;
} catch (const std::logic_error&) {
move_assignment_invalidated = true;
}
EXPECT(move_assignment_invalidated);
EXPECT(destination.id_alive(transferred_id));
EXPECT(moved.empty());
moved.push_back(2'000);
EXPECT(moved.contains(2'000));
Sequence copy_source;
copy_source.push_back(99);
auto stale_after_copy_assignment = destination[0];
destination = copy_source;
bool copy_assignment_invalidated = false;
try {
stale_after_copy_assignment = 9;
} catch (const std::logic_error&) {
copy_assignment_invalidated = true;
}
EXPECT(copy_assignment_invalidated);
EXPECT(destination.size() == 1);
EXPECT(destination.contains(99));
}
void hash_backward_shift_handles_wraparound_and_duplicates() {
using Sequence = uc::AdaptiveSequence<std::uint32_t, true,
uc::ResizePolicy, WraparoundHash>;
Sequence values;
std::vector<Sequence::stable_id> ids;
for (std::uint32_t value = 0; value < 12; ++value) {
values.push_back(value);
ids.push_back(values.id_at(values.size() - 1));
}
values.push_back(5);
const auto duplicate_id = values.id_at(values.size() - 1);
const auto buckets = values.hash_bucket_count();
// Every distinct key starts in the final bucket, so the probe chain wraps
// through bucket zero. Removing a middle key must compact that chain
// without losing later keys or the duplicate list.
values.erase_by_id(ids[3]);
EXPECT(!values.id_alive(ids[3]));
for (std::uint32_t value = 0; value < 12; ++value) {
EXPECT(values.contains(value) == (value != 3));
}
EXPECT(values.find_all(5).size() == 2);
EXPECT(values.id_alive(ids[5]));
EXPECT(values.id_alive(duplicate_id));
values.erase_by_id(ids[5]);
EXPECT(values.contains(5));
EXPECT(values.find_all_ids(5)
== std::vector<Sequence::stable_id>({duplicate_id}));
EXPECT(values.hash_bucket_count() == buckets);
}
void indexed_proxy_detects_structural_invalidation() {
uc::AdaptiveSequence<std::uint32_t, true> values({8, 4, 2});
values.push_back(10);
@@ -278,47 +436,66 @@ void tiered_shape_rebuild_preserves_order() {
EXPECT(values.policy().telemetry().tiered_directory_rebuilds == 2);
}
void adaptation_is_deferred_until_a_safe_boundary() {
using sequence_type = uc::AdaptiveSequence<
std::uint32_t, false, DeferredTransitionPolicy>;
void capacity_boundaries_control_mode_and_geometry() {
uc::ResizePolicyConfig policy_config;
policy_config.minimum_tiered_size = 8;
uc::ResizePolicy policy(policy_config, {4, 4, 3});
uc::AdaptiveSequence<std::uint32_t> values({4, 4, 3}, policy);
sequence_type explicit_maintenance({8, 4, 2}, DeferredTransitionPolicy{});
for (std::uint32_t value = 0; value < 4; ++value) {
explicit_maintenance.push_back(value);
for (std::uint32_t value = 0; value < 8; ++value) {
values.push_back(value);
}
explicit_maintenance.insert(2, 99);
EXPECT(explicit_maintenance.mode() == uc::StorageMode::vector);
EXPECT(explicit_maintenance.policy().decision_ready());
EXPECT(explicit_maintenance.policy().transitions == 0);
EXPECT(explicit_maintenance.adapt_now());
EXPECT(explicit_maintenance.mode() == uc::StorageMode::tiered);
EXPECT(explicit_maintenance.policy().decisions == 1);
EXPECT(explicit_maintenance.policy().transitions == 1);
EXPECT(explicit_maintenance.policy().last_from == uc::StorageMode::vector);
EXPECT(explicit_maintenance.policy().last_to == uc::StorageMode::tiered);
EXPECT(!explicit_maintenance.adapt_now());
EXPECT(!explicit_maintenance.adapt_now());
EXPECT(explicit_maintenance.policy().decisions == 1);
EXPECT(explicit_maintenance.policy().transitions == 1);
expect_equal(explicit_maintenance,
std::vector<std::uint32_t>({0, 1, 99, 2, 3}));
EXPECT(values.capacity() == 8);
EXPECT(values.mode() == uc::StorageMode::vector);
sequence_type next_mutation({8, 4, 2}, DeferredTransitionPolicy{});
for (std::uint32_t value = 0; value < 4; ++value) {
next_mutation.push_back(value);
// The ninth element is the next x2 capacity boundary. The representation
// and sqrt(N) leaf are selected as part of that single rebuild.
values.push_back(8);
EXPECT(values.capacity() == 16);
EXPECT(values.mode() == uc::StorageMode::tiered);
EXPECT(values.tiered_config().leaf_capacity == 4);
const auto leaf_at_16 = values.tiered_config().leaf_capacity;
for (std::uint32_t value = 9; value < 16; ++value) {
values.push_back(value);
EXPECT(values.tiered_config().leaf_capacity == leaf_at_16);
}
next_mutation.insert(1, 77);
EXPECT(next_mutation.mode() == uc::StorageMode::vector);
EXPECT(next_mutation.policy().transitions == 0);
// The pending recommendation is applied before this append; it is not
// applied after the preceding, already committed insertion.
next_mutation.push_back(4);
EXPECT(next_mutation.mode() == uc::StorageMode::tiered);
EXPECT(next_mutation.policy().decisions == 1);
EXPECT(next_mutation.policy().transitions == 1);
expect_equal(next_mutation,
std::vector<std::uint32_t>({0, 77, 1, 2, 3, 4}));
values.push_back(16);
EXPECT(values.capacity() == 32);
EXPECT(values.mode() == uc::StorageMode::tiered);
EXPECT(values.tiered_config().leaf_capacity == 5);
// Shrink is exactly one /2 step at 12.5% occupancy. At N=4 the same
// rebuild crosses the cutoff in the opposite direction.
while (values.size() > 4) {
values.erase(values.size() - 1);
}
EXPECT(values.capacity() == 16);
EXPECT(values.mode() == uc::StorageMode::vector);
expect_equal(values, std::vector<std::uint32_t>({0, 1, 2, 3}));
EXPECT(!values.adapt_now());
EXPECT(values.mode() == uc::StorageMode::vector);
}
void calibrated_default_cutoff_is_applied_at_resize() {
uc::AdaptiveSequence<std::uint32_t> values;
for (std::uint32_t value = 0; value < 4'096; ++value) {
values.push_back(value);
}
EXPECT(values.capacity() == 4'096);
EXPECT(values.mode() == uc::StorageMode::vector);
values.push_back(4'096);
EXPECT(values.capacity() == 8'192);
EXPECT(values.mode() == uc::StorageMode::tiered);
EXPECT(values.tiered_config().leaf_capacity == 65);
while (values.size() > 1'024) {
values.erase(values.size() - 1);
}
EXPECT(values.capacity() == 4'096);
EXPECT(values.mode() == uc::StorageMode::vector);
}
std::optional<uc::AdaptationDecision> shape_decision(
@@ -540,69 +717,42 @@ void policy_changes_shape_with_scale_and_locality() {
EXPECT(depth->tiered_config.directory_levels == 3);
}
void container_applies_same_mode_shape_decision() {
uc::AdaptationConfig config;
config.evaluation_interval = 64;
config.minimum_observations = 64;
config.minimum_residency_operations = 0;
config.minimum_shape_residency_operations = 0;
config.minimum_shape_edit_fraction = 0.0;
config.required_confirmation_windows = 1;
config.minimum_tiered_size = 1;
config.forecast_operations = 1'000'000;
config.tiered_rebuild_safety = 0.0;
config.minimum_shape_improvement = 0.0;
config.vector_move_unit = 1.0;
const uc::TieredConfig initial_shape{1024, 64, 4};
uc::AdaptiveSequence<std::uint32_t> values(
initial_shape, uc::CostModelPolicy(config, initial_shape));
for (std::uint32_t i = 0; i < 10'000; ++i) {
values.push_back(i);
}
values.force_tiered_mode(initial_shape);
values.enable_auto_mode();
void indexed_hash_capacity_tracks_container_capacity() {
uc::ResizePolicyConfig policy_config;
policy_config.minimum_tiered_size = 32;
uc::AdaptiveSequence<std::uint32_t, true> values(
{8, 8, 3}, uc::ResizePolicy(policy_config, {8, 8, 3}));
for (std::size_t i = 0;
i < 256 && values.policy().telemetry().tiered_rebuilds == 0; ++i) {
const auto position = (i * 104'729 + 17) % values.size();
values.insert(position, 0xabcdefu);
values.erase(position);
auto previous_capacity = values.capacity();
auto previous_buckets = values.hash_bucket_count();
for (std::uint32_t value = 0; value < 257; ++value) {
values.push_back(value * 16);
EXPECT(values.contains(value * 16));
if (values.capacity() == previous_capacity) {
EXPECT(values.hash_bucket_count() == previous_buckets);
} else {
EXPECT(values.capacity() == 1
|| values.capacity() == previous_capacity * 2);
EXPECT(values.hash_bucket_count() >= previous_buckets);
previous_capacity = values.capacity();
previous_buckets = values.hash_bucket_count();
}
}
EXPECT(values.mode() == uc::StorageMode::tiered);
EXPECT(values.policy().telemetry().tiered_rebuilds == 1);
EXPECT(values.tiered_config().leaf_capacity <= 128);
for (std::size_t i = 0; i < values.size(); ++i) {
EXPECT(values[i] == i);
// These keys share low hash bits and exercise backward-shift deletion in
// the linear-probing table without an intermediate rehash.
for (std::uint32_t value = 0; value < 257; value += 2) {
EXPECT(values.erase_one(value * 16));
}
for (std::uint32_t value = 0; value < 257; ++value) {
EXPECT(values.contains(value * 16) == (value % 2 != 0));
}
}
void policy_switches_for_sustained_middle_edits() {
uc::TieredConfig tiered{32, 16, 3};
uc::AdaptationConfig adaptation;
adaptation.evaluation_interval = 64;
adaptation.minimum_observations = 64;
adaptation.minimum_residency_operations = 0;
adaptation.minimum_tiered_size = 128;
adaptation.forecast_operations = 16'384;
adaptation.vector_to_tiered_safety = 1.0;
// Make the decision boundary deliberately small; this test checks policy
// plumbing, while machine-calibrated constants are validated by benchmarks.
adaptation.vector_move_unit = 0.20;
adaptation.tiered_move_unit = 0.20;
adaptation.tiered_read_base = 2.0;
uc::AdaptiveSequence<std::uint32_t> values(
tiered, uc::CostModelPolicy(adaptation, tiered));
for (std::uint32_t i = 0; i < 512; ++i) {
values.push_back(i);
const auto buckets_before_shrink = values.hash_bucket_count();
while (values.size() > values.capacity() / 8) {
values.erase(values.size() - 1);
}
values.enable_auto_mode();
for (std::size_t i = 0; i < 256 && values.mode() != uc::StorageMode::tiered; ++i) {
values.insert(values.size() / 2, 7);
values.erase(values.size() / 2);
}
EXPECT(values.mode() == uc::StorageMode::tiered);
EXPECT(values.hash_bucket_count() <= buckets_before_shrink);
}
} // namespace
@@ -611,17 +761,20 @@ int main() {
ring_block_offsets();
randomized_differential();
indexed_duplicates_and_ids();
indexed_ids_reuse_slots_without_reviving_stale_handles();
moved_from_sequences_are_reusable_and_assignment_invalidates_proxies();
hash_backward_shift_handles_wraparound_and_duplicates();
indexed_proxy_detects_structural_invalidation();
iterator_and_contiguous_contract();
non_trivial_values();
tiered_shape_rebuild_preserves_order();
adaptation_is_deferred_until_a_safe_boundary();
capacity_boundaries_control_mode_and_geometry();
calibrated_default_cutoff_is_applied_at_resize();
forecast_horizon_is_bounded_by_observed_evidence();
confirmation_requires_consecutive_supporting_windows();
stationary_phases_do_not_thrash();
policy_changes_shape_with_scale_and_locality();
container_applies_same_mode_shape_decision();
policy_switches_for_sustained_middle_edits();
indexed_hash_capacity_tracks_container_capacity();
if (failures != 0) {
std::cerr << failures << " test assertion(s) failed\n";