New adaptation policy

include/universal_container/adaptation_policy.hpp

@@ -151,6 +151,89 @@ struct AdaptationTelemetry {
     std::size_t last_evidence_windows = 0;
 };
 
+// The production policy is deliberately small.  It does not try to infer a
+// workload phase from every operation; AdaptiveSequence consults these limits
+// only when its logical capacity grows or shrinks.  The crossover value is a
+// calibratable constant and is updated from the focused benchmark rather than
+// from the retired benchmark matrix.
+struct ResizePolicyConfig {
+    std::size_t minimum_tiered_size = 4 * 1024;
+    std::size_t shrink_denominator = 8; // shrink at 12.5% occupancy
+};
+
+class ResizePolicy {
+public:
+    explicit ResizePolicy(ResizePolicyConfig config = {}, TieredConfig tiered = {}) noexcept
+        : config_(normalize(config)), active_tiered_(normalize_tiered(tiered)) {}
+
+    void set_tiered_config(TieredConfig tiered) noexcept {
+        active_tiered_ = normalize_tiered(tiered);
+    }
+
+    // Kept as no-op compatibility hooks so a custom CostModelPolicy can still
+    // be substituted for experiments without putting policy work on the
+    // default read/edit paths.
+    void observe(const OperationSample&) noexcept {}
+    [[nodiscard]] bool decision_ready() const noexcept { return false; }
+    [[nodiscard]] std::optional<AdaptationDecision>
+    recommended_decision(StorageMode, std::size_t, TieredConfig) noexcept {
+        return std::nullopt;
+    }
+
+    void on_transition(StorageMode from,
+                       StorageMode to,
+                       TieredConfig active_config) noexcept {
+        const auto normalized = normalize_tiered(active_config);
+        if (from == StorageMode::vector && to == StorageMode::tiered) {
+            ++telemetry_.vector_to_tiered;
+        } else if (from == StorageMode::tiered && to == StorageMode::vector) {
+            ++telemetry_.tiered_to_vector;
+        } else if (from == StorageMode::tiered && to == StorageMode::tiered) {
+            ++telemetry_.tiered_rebuilds;
+            if (normalized.leaf_capacity == active_tiered_.leaf_capacity) {
+                ++telemetry_.tiered_directory_rebuilds;
+            } else {
+                ++telemetry_.tiered_leaf_rebuilds;
+            }
+        }
+        active_tiered_ = normalized;
+    }
+
+    void on_transition(StorageMode from, StorageMode to) noexcept {
+        on_transition(from, to, active_tiered_);
+    }
+
+    void reset() noexcept { telemetry_ = {}; }
+
+    [[nodiscard]] const ResizePolicyConfig& config() const noexcept { return config_; }
+    [[nodiscard]] const AdaptationTelemetry& telemetry() const noexcept {
+        return telemetry_;
+    }
+
+private:
+    static ResizePolicyConfig normalize(ResizePolicyConfig config) noexcept {
+        config.minimum_tiered_size = std::max<std::size_t>(1,
+            config.minimum_tiered_size);
+        config.shrink_denominator = std::max<std::size_t>(2,
+            config.shrink_denominator);
+        return config;
+    }
+
+    static TieredConfig normalize_tiered(TieredConfig config) noexcept {
+        config.leaf_capacity = std::clamp<std::size_t>(
+            config.leaf_capacity, 4, 1U << 20U);
+        config.directory_fanout = std::clamp<std::size_t>(
+            config.directory_fanout, 2, 1U << 16U);
+        config.directory_levels = std::clamp<std::size_t>(
+            config.directory_levels, 1, 8);
+        return config;
+    }
+
+    ResizePolicyConfig config_{};
+    TieredConfig active_tiered_{};
+    AdaptationTelemetry telemetry_{};
+};
+
 class CostModelPolicy {
 public:
     static constexpr std::size_t candidate_count = 5;

include/universal_container/adaptive_sequence.hpp

@@ -10,6 +10,7 @@
 #include <concepts>
 #include <cstddef>
 #include <cstdint>
+#include <cmath>
 #include <functional>
 #include <iterator>
 #include <limits>
@@ -25,7 +26,7 @@ namespace uc {
 
 template <class T,
           bool HashIndexEnabled = false,
-          class AdaptationPolicy = CostModelPolicy,
+          class AdaptationPolicy = ResizePolicy,
           class Hash = std::hash<T>,
           class Equal = std::equal_to<T>>
 class AdaptiveSequence {
@@ -54,8 +55,10 @@ private:
     struct IdMetadata {
         internal_id previous = invalid_internal_id;
         internal_id next = invalid_internal_id;
+        internal_id free_next = invalid_internal_id;
         std::uint32_t primary = 0;   // vector index or stable leaf id
         std::uint32_t secondary = 0; // local leaf index
+        std::uint32_t generation = 0;
         std::uint8_t flags = 0;
     };
 
@@ -65,6 +68,7 @@ private:
     struct IndexedState {
         detail::FlatDuplicateIndex<T, Hash, Equal> values;
         std::vector<IdMetadata> ids;
+        internal_id free_head = invalid_internal_id;
     };
 
     struct NoIndexState {};
@@ -237,9 +241,53 @@ public:
     }
 
     AdaptiveSequence(const AdaptiveSequence&) = default;
-    AdaptiveSequence(AdaptiveSequence&&) noexcept = default;
-    AdaptiveSequence& operator=(const AdaptiveSequence&) = default;
-    AdaptiveSequence& operator=(AdaptiveSequence&&) noexcept = default;
+
+    AdaptiveSequence(AdaptiveSequence&& other) noexcept
+        : tiered_config_(std::move(other.tiered_config_)),
+          policy_(std::move(other.policy_)),
+          storage_(std::move(other.storage_)),
+          index_state_(std::move(other.index_state_)),
+          equal_(std::move(other.equal_)),
+          residency_(other.residency_),
+          read_adaptation_(other.read_adaptation_),
+          logical_capacity_(other.logical_capacity_),
+          read_sample_rate_(other.read_sample_rate_),
+          read_sample_countdown_(other.read_sample_countdown_),
+          edit_sample_rate_(other.edit_sample_rate_),
+          edit_sample_countdown_(other.edit_sample_countdown_),
+          generation_(other.generation_) {
+        other.reset_after_move();
+    }
+
+    AdaptiveSequence& operator=(const AdaptiveSequence& other) {
+        if (this != &other) {
+            AdaptiveSequence replacement(other);
+            *this = std::move(replacement);
+        }
+        return *this;
+    }
+
+    AdaptiveSequence& operator=(AdaptiveSequence&& other) noexcept {
+        if (this == &other) {
+            return *this;
+        }
+        const auto invalidated_generation = generation_ + 1;
+        tiered_config_ = std::move(other.tiered_config_);
+        policy_ = std::move(other.policy_);
+        storage_ = std::move(other.storage_);
+        index_state_ = std::move(other.index_state_);
+        equal_ = std::move(other.equal_);
+        residency_ = other.residency_;
+        read_adaptation_ = other.read_adaptation_;
+        logical_capacity_ = other.logical_capacity_;
+        read_sample_rate_ = other.read_sample_rate_;
+        read_sample_countdown_ = other.read_sample_countdown_;
+        edit_sample_rate_ = other.edit_sample_rate_;
+        edit_sample_countdown_ = other.edit_sample_countdown_;
+        generation_ = invalidated_generation;
+        other.reset_after_move();
+        return *this;
+    }
     ~AdaptiveSequence() = default;
 
     [[nodiscard]] size_type size() const noexcept {
@@ -249,6 +297,10 @@ public:
     }
 
     [[nodiscard]] bool empty() const noexcept { return size() == 0; }
+    // Logical capacity is shared by both backends.  It is the only trigger for
+    // automatic representation/geometry changes; TieredStorage's internal
+    // leaf slack is intentionally not exposed as container capacity.
+    [[nodiscard]] size_type capacity() const noexcept { return logical_capacity_; }
     [[nodiscard]] StorageMode mode() const noexcept {
         return std::holds_alternative<vector_storage>(storage_)
             ? StorageMode::vector
@@ -258,13 +310,7 @@ public:
     [[nodiscard]] const TieredConfig& tiered_config() const noexcept { return tiered_config_; }
 
     reference operator[](size_type index) {
-        if (read_adaptation_ == ReadAdaptationMode::eager_nonconst) {
-            apply_pending_adaptation();
-        }
         observe_random_read();
-        if (read_adaptation_ == ReadAdaptationMode::eager_nonconst) {
-            apply_pending_adaptation();
-        }
         if constexpr (HashIndexEnabled) {
             return reference_proxy(*this, index);
         } else {
@@ -295,7 +341,6 @@ public:
 
     void erase(size_type index) {
         check_index(index);
-        apply_pending_adaptation();
         const auto old_size = size();
         const auto id = record_id(record_at_unchecked(index));
         if constexpr (HashIndexEnabled) {
@@ -311,13 +356,11 @@ public:
             (void)std::get<tiered_storage>(storage_).erase(index, relocate);
         }
         if constexpr (HashIndexEnabled) {
-            auto& metadata = index_state_.ids[static_cast<size_type>(id)];
-            metadata.flags &= static_cast<std::uint8_t>(~alive_flag);
-            metadata.previous = invalid_internal_id;
-            metadata.next = invalid_internal_id;
+            release_id_slot(id);
         }
         ++generation_;
         observe_structural_edit(OperationKind::erase, old_size, index);
+        shrink_after_erase();
     }
 
     void set(size_type index, const T& value) { set_impl(index, T(value)); }
@@ -325,14 +368,11 @@ public:
 
     void clear() {
         if constexpr (HashIndexEnabled) {
-            for (auto& metadata : index_state_.ids) {
-                metadata.flags &= static_cast<std::uint8_t>(~alive_flag);
-                metadata.previous = invalid_internal_id;
-                metadata.next = invalid_internal_id;
-            }
             index_state_.values.clear();
+            rebuild_free_id_list();
         }
         storage_.template emplace<vector_storage>();
+        logical_capacity_ = 0;
         residency_ = ResidencyMode::automatic;
         policy_.reset();
         read_sample_countdown_ = read_sample_rate_;
@@ -342,9 +382,12 @@ public:
 
     void reserve(size_type capacity) {
         force_vector_mode();
-        std::get<vector_storage>(storage_).reserve(capacity);
+        const auto requested = std::max(capacity, size());
+        std::get<vector_storage>(storage_).reserve(requested);
+        logical_capacity_ = std::max(logical_capacity_, requested);
         if constexpr (HashIndexEnabled) {
-            const auto additional = capacity > size() ? capacity - size() : 0;
+            index_state_.values.reserve_for_elements(logical_capacity_);
+            const auto additional = requested > size() ? requested - size() : 0;
             index_state_.ids.reserve(index_state_.ids.size() + additional);
         }
         ++generation_;
@@ -364,14 +407,23 @@ public:
             if (!entry || entry->head == invalid_internal_id) {
                 return std::nullopt;
             }
-            return resolve_id(entry->head);
+            return resolve_internal_id(entry->head);
         } else {
-            for (size_type i = 0; i < size(); ++i) {
-                if (equal_(value_at_unchecked(i), value)) {
-                    return i;
+            if (mode() == StorageMode::vector) {
+                const auto& values = std::get<vector_storage>(storage_);
+                for (size_type i = 0; i < values.size(); ++i) {
+                    if (equal_(record_value(values[i]), value)) {
+                        return i;
+                    }
                 }
+                return std::nullopt;
             }
-            return std::nullopt;
+            const auto found = std::get<tiered_storage>(storage_).find_if(
+                [&](const record_type& record) {
+                    return equal_(record_value(record), value);
+                });
+            return found == tiered_storage::npos
+                ? std::nullopt : std::optional<size_type>(found);
         }
     }
 
@@ -385,15 +437,26 @@ public:
             result.reserve(entry->count);
             auto id = entry->head;
             while (id != invalid_internal_id) {
-                result.push_back(resolve_id(id));
+                result.push_back(resolve_internal_id(id));
                 id = index_state_.ids[static_cast<size_type>(id)].next;
             }
             std::sort(result.begin(), result.end());
         } else {
-            for (size_type i = 0; i < size(); ++i) {
-                if (equal_(value_at_unchecked(i), value)) {
-                    result.push_back(i);
+            if (mode() == StorageMode::vector) {
+                const auto& values = std::get<vector_storage>(storage_);
+                for (size_type i = 0; i < values.size(); ++i) {
+                    if (equal_(record_value(values[i]), value)) {
+                        result.push_back(i);
+                    }
                 }
+            } else {
+                std::get<tiered_storage>(storage_).for_each_match(
+                    [&](const record_type& record) {
+                        return equal_(record_value(record), value);
+                    },
+                    [&](size_type index, const record_type&) {
+                        result.push_back(index);
+                    });
             }
         }
         return result;
@@ -411,7 +474,7 @@ public:
         result.reserve(entry->count);
         auto id = entry->head;
         while (id != invalid_internal_id) {
-            result.push_back(static_cast<stable_id>(id));
+            result.push_back(stable_id_for_slot(id));
             id = index_state_.ids[static_cast<size_type>(id)].next;
         }
         return result;
@@ -432,11 +495,11 @@ public:
             if (!entry) {
                 return 0;
             }
-            std::vector<internal_id> ids;
+            std::vector<stable_id> ids;
             ids.reserve(entry->count);
             auto id = entry->head;
             while (id != invalid_internal_id) {
-                ids.push_back(id);
+                ids.push_back(stable_id_for_slot(id));
                 id = index_state_.ids[static_cast<size_type>(id)].next;
             }
             for (const auto current : ids) {
@@ -460,14 +523,22 @@ public:
     [[nodiscard]] stable_id id_at(size_type index) const
         requires(HashIndexEnabled) {
         check_index(index);
-        return static_cast<stable_id>(record_at_unchecked(index).id);
+        return stable_id_for_slot(record_at_unchecked(index).id);
     }
 
     [[nodiscard]] bool id_alive(stable_id id) const noexcept
         requires(HashIndexEnabled) {
-        return id <= static_cast<stable_id>(std::numeric_limits<internal_id>::max())
-            && id < index_state_.ids.size()
-            && (index_state_.ids[static_cast<size_type>(id)].flags & alive_flag) != 0;
+        if (id == invalid_id) {
+            return false;
+        }
+        const auto slot = slot_from_stable_id(id);
+        if (slot == invalid_internal_id
+            || static_cast<size_type>(slot) >= index_state_.ids.size()) {
+            return false;
+        }
+        const auto& metadata = index_state_.ids[static_cast<size_type>(slot)];
+        return metadata.generation == generation_from_stable_id(id)
+            && (metadata.flags & alive_flag) != 0;
     }
 
     void erase_by_id(stable_id id)
@@ -532,12 +603,11 @@ public:
         return read_adaptation_;
     }
 
-    bool adapt_now() {
-        if (residency_ != ResidencyMode::automatic) {
-            return false;
-        }
-        return apply_pending_adaptation();
-    }
+    // Automatic rebuilds are capacity-boundary operations.  This compatibility
+    // hook therefore never changes the representation between two resize
+    // events; explicit force_* calls remain available for callers that need an
+    // immediate manual conversion.
+    bool adapt_now() noexcept { return false; }
 
     [[nodiscard]] T* data() noexcept
         requires(!HashIndexEnabled) {
@@ -625,10 +695,29 @@ public:
         return bytes;
     }
 
+    [[nodiscard]] std::size_t hash_bucket_count() const noexcept
+        requires(HashIndexEnabled) {
+        return index_state_.values.bucket_count();
+    }
+
     [[nodiscard]] AdaptationPolicy& policy() noexcept { return policy_; }
     [[nodiscard]] const AdaptationPolicy& policy() const noexcept { return policy_; }
 
 private:
+    void reset_after_move() noexcept {
+        storage_.template emplace<vector_storage>();
+        if constexpr (HashIndexEnabled) {
+            index_state_.values.reset_after_move();
+            index_state_.ids.clear();
+            index_state_.free_head = invalid_internal_id;
+        }
+        residency_ = ResidencyMode::automatic;
+        logical_capacity_ = 0;
+        read_sample_countdown_ = read_sample_rate_;
+        edit_sample_countdown_ = edit_sample_rate_;
+        ++generation_;
+    }
+
     static void set_location(IdMetadata& metadata,
                              bool tiered,
                              std::uint32_t primary,
@@ -667,14 +756,88 @@ private:
         }
     }
 
+    static constexpr unsigned stable_id_slot_bits =
+        std::numeric_limits<internal_id>::digits;
+    static_assert(stable_id_slot_bits == 32);
+
+    [[nodiscard]] static internal_id slot_from_stable_id(stable_id id) noexcept {
+        return static_cast<internal_id>(id);
+    }
+
+    [[nodiscard]] static std::uint32_t generation_from_stable_id(
+        stable_id id) noexcept {
+        return static_cast<std::uint32_t>(id >> stable_id_slot_bits);
+    }
+
+    [[nodiscard]] stable_id stable_id_for_slot(internal_id slot) const noexcept
+        requires(HashIndexEnabled) {
+        const auto generation =
+            index_state_.ids[static_cast<size_type>(slot)].generation;
+        return (static_cast<stable_id>(generation) << stable_id_slot_bits)
+             | static_cast<stable_id>(slot);
+    }
+
+    [[nodiscard]] internal_id acquire_id_slot()
+        requires(HashIndexEnabled) {
+        if (index_state_.free_head != invalid_internal_id) {
+            const auto id = index_state_.free_head;
+            auto& metadata = index_state_.ids[static_cast<size_type>(id)];
+            index_state_.free_head = metadata.free_next;
+            assert(metadata.generation
+                   < std::numeric_limits<std::uint32_t>::max());
+            const auto next_generation = metadata.generation + 1;
+            metadata = {};
+            metadata.generation = next_generation;
+            return id;
+        }
+
+        if (index_state_.ids.size() >= static_cast<size_type>(invalid_internal_id)) {
+            throw std::length_error("stable id slot space exhausted");
+        }
+        const auto id = static_cast<internal_id>(index_state_.ids.size());
+        index_state_.ids.emplace_back();
+        return id;
+    }
+
+    void release_id_slot(internal_id id) noexcept
+        requires(HashIndexEnabled) {
+        assert(id != invalid_internal_id);
+        assert(static_cast<size_type>(id) < index_state_.ids.size());
+        auto& metadata = index_state_.ids[static_cast<size_type>(id)];
+        const auto generation = metadata.generation;
+        metadata = {};
+        metadata.generation = generation;
+        if (generation != std::numeric_limits<std::uint32_t>::max()) {
+            metadata.free_next = index_state_.free_head;
+            index_state_.free_head = id;
+        }
+    }
+
+    void rebuild_free_id_list() noexcept
+        requires(HashIndexEnabled) {
+        index_state_.free_head = invalid_internal_id;
+        for (size_type index = index_state_.ids.size(); index != 0; --index) {
+            const auto id = static_cast<internal_id>(index - 1);
+            auto& metadata = index_state_.ids[index - 1];
+            const auto generation = metadata.generation;
+            metadata = {};
+            metadata.generation = generation;
+            if (generation != std::numeric_limits<std::uint32_t>::max()) {
+                metadata.free_next = index_state_.free_head;
+                index_state_.free_head = id;
+            }
+        }
+    }
+
     record_type make_record(T value) {
         if constexpr (HashIndexEnabled) {
-            if (index_state_.ids.size() >= static_cast<size_type>(invalid_internal_id)) {
-                throw std::length_error("stable id space exhausted");
+            const auto id = acquire_id_slot();
+            try {
+                return IndexedRecord{std::move(value), id};
+            } catch (...) {
+                release_id_slot(id);
+                throw;
             }
-            const auto id = static_cast<internal_id>(index_state_.ids.size());
-            index_state_.ids.emplace_back();
-            return IndexedRecord{std::move(value), id};
         } else {
             return value;
         }
@@ -707,23 +870,40 @@ private:
     }
 
     void push_back_impl(T value) {
-        apply_pending_adaptation();
         const auto old_size = size();
+        grow_before_insert(old_size + 1);
         auto record = make_record(std::move(value));
         const auto id = record_id(record);
         auto relocate = relocation_callback();
-        if (mode() == StorageMode::vector) {
-            auto& values = std::get<vector_storage>(storage_);
-            values.push_back(std::move(record));
-            refresh_vector_locations(old_size);
+        if constexpr (HashIndexEnabled) {
+            try {
+                // Keep an unmoved lookup key so a failed storage insertion can
+                // roll the duplicate chain back before the slot is recycled.
+                T rollback_key(record.value);
+                link_value(record.value, id);
+                try {
+                    if (mode() == StorageMode::vector) {
+                        auto& values = std::get<vector_storage>(storage_);
+                        values.push_back(std::move(record));
+                        refresh_vector_locations(old_size);
+                    } else {
+                        std::get<tiered_storage>(storage_).push_back(
+                            std::move(record), relocate);
+                    }
+                } catch (...) {
+                    unlink_value(rollback_key, id);
+                    throw;
+                }
+                index_state_.ids[static_cast<size_type>(id)].flags |= alive_flag;
+            } catch (...) {
+                release_id_slot(id);
+                throw;
+            }
+        } else if (mode() == StorageMode::vector) {
+            std::get<vector_storage>(storage_).push_back(std::move(record));
         } else {
             std::get<tiered_storage>(storage_).push_back(std::move(record), relocate);
         }
-        if constexpr (HashIndexEnabled) {
-            auto& metadata = index_state_.ids[static_cast<size_type>(id)];
-            metadata.flags |= alive_flag;
-            link_value(value_at_unchecked(old_size), id);
-        }
         ++generation_;
         policy_.observe({OperationKind::append, old_size, old_size, 1, sizeof(T)});
     }
@@ -732,23 +912,41 @@ private:
         if (index > size()) {
             throw std::out_of_range("AdaptiveSequence insertion index out of range");
         }
-        apply_pending_adaptation();
         const auto old_size = size();
+        grow_before_insert(old_size + 1);
         auto record = make_record(std::move(value));
         const auto id = record_id(record);
         auto relocate = relocation_callback();
-        if (mode() == StorageMode::vector) {
+        if constexpr (HashIndexEnabled) {
+            try {
+                T rollback_key(record.value);
+                link_value(record.value, id);
+                try {
+                    if (mode() == StorageMode::vector) {
+                        auto& values = std::get<vector_storage>(storage_);
+                        values.insert(values.begin() + static_cast<difference_type>(index),
+                                      std::move(record));
+                        refresh_vector_locations(index);
+                    } else {
+                        std::get<tiered_storage>(storage_).insert(
+                            index, std::move(record), relocate);
+                    }
+                } catch (...) {
+                    unlink_value(rollback_key, id);
+                    throw;
+                }
+                index_state_.ids[static_cast<size_type>(id)].flags |= alive_flag;
+            } catch (...) {
+                release_id_slot(id);
+                throw;
+            }
+        } else if (mode() == StorageMode::vector) {
             auto& values = std::get<vector_storage>(storage_);
-            values.insert(values.begin() + static_cast<difference_type>(index), std::move(record));
-            refresh_vector_locations(index);
+            values.insert(values.begin() + static_cast<difference_type>(index),
+                          std::move(record));
         } else {
             std::get<tiered_storage>(storage_).insert(index, std::move(record), relocate);
         }
-        if constexpr (HashIndexEnabled) {
-            auto& metadata = index_state_.ids[static_cast<size_type>(id)];
-            metadata.flags |= alive_flag;
-            link_value(value_at_unchecked(index), id);
-        }
         ++generation_;
         observe_structural_edit(OperationKind::insert, old_size, index);
     }
@@ -837,13 +1035,28 @@ private:
 
     [[nodiscard]] size_type resolve_id(stable_id id) const
         requires(HashIndexEnabled) {
-        if (id >= index_state_.ids.size()) {
+        if (id == invalid_id) {
             throw std::out_of_range("unknown stable id");
         }
-        const auto& metadata = index_state_.ids[static_cast<size_type>(id)];
-        if ((metadata.flags & alive_flag) == 0) {
+        const auto slot = slot_from_stable_id(id);
+        if (slot == invalid_internal_id
+            || static_cast<size_type>(slot) >= index_state_.ids.size()) {
+            throw std::out_of_range("unknown stable id");
+        }
+        const auto& metadata = index_state_.ids[static_cast<size_type>(slot)];
+        if (metadata.generation != generation_from_stable_id(id)
+            || (metadata.flags & alive_flag) == 0) {
             throw std::out_of_range("stable id no longer refers to an element");
         }
+        return resolve_internal_id(slot);
+    }
+
+    [[nodiscard]] size_type resolve_internal_id(internal_id id) const noexcept
+        requires(HashIndexEnabled) {
+        assert(id != invalid_internal_id);
+        assert(static_cast<size_type>(id) < index_state_.ids.size());
+        const auto& metadata = index_state_.ids[static_cast<size_type>(id)];
+        assert((metadata.flags & alive_flag) != 0);
         if ((metadata.flags & tiered_flag) == 0) {
             return metadata.primary;
         }
@@ -852,51 +1065,171 @@ private:
             metadata.secondary);
     }
 
-    bool apply_pending_adaptation() {
-        if (residency_ != ResidencyMode::automatic || !policy_.decision_ready()) {
-            return false;
+    [[nodiscard]] static size_type ceil_sqrt(size_type value) noexcept {
+        if (value <= 1) {
+            return value;
+        }
+        auto root = static_cast<size_type>(std::sqrt(static_cast<long double>(value)));
+        while (root > value / root) {
+            --root;
+        }
+        while (root < value / root
+               || (root <= value / root && root * root < value)) {
+            ++root;
+        }
+        return root;
+    }
+
+    [[nodiscard]] size_type minimum_tiered_size() const noexcept {
+        if constexpr (requires(const AdaptationPolicy& p) {
+                          p.config().minimum_tiered_size;
+                      }) {
+            return std::max<size_type>(1, policy_.config().minimum_tiered_size);
+        } else {
+            return 4 * 1024;
+        }
+    }
+
+    [[nodiscard]] size_type shrink_denominator() const noexcept {
+        if constexpr (requires(const AdaptationPolicy& p) {
+                          p.config().shrink_denominator;
+                      }) {
+            return std::max<size_type>(2, policy_.config().shrink_denominator);
+        } else {
+            return 8;
+        }
+    }
+
+    [[nodiscard]] StorageMode automatic_mode_for(size_type element_count) const noexcept {
+        return element_count >= minimum_tiered_size()
+            ? StorageMode::tiered : StorageMode::vector;
+    }
+
+    [[nodiscard]] TieredConfig geometry_for(size_type element_count) const noexcept {
+        auto result = tiered_config_;
+        result.leaf_capacity = std::max<size_type>(4, ceil_sqrt(element_count));
+        return result;
+    }
+
+    void grow_before_insert(size_type required) {
+        if (required <= logical_capacity_) {
+            return;
+        }
+        size_type next = logical_capacity_ == 0 ? 1 : logical_capacity_;
+        while (next < required) {
+            if (next > std::numeric_limits<size_type>::max() / 2) {
+                next = required;
+                break;
+            }
+            next *= 2;
+        }
+        rebuild_for_capacity(next, required);
+    }
+
+    void shrink_after_erase() noexcept {
+        if (logical_capacity_ <= 1
+            || size() > logical_capacity_ / shrink_denominator()) {
+            return;
+        }
+        const auto next = std::max(size(), logical_capacity_ / 2);
+        // The erase has already committed.  Capacity shrink is an automatic
+        // optimization and must not turn that successful erase into a thrown
+        // operation.  Copyable records keep their source intact on failure;
+        // nothrow-movable records use preallocated rebuild paths below.
+        if constexpr (std::is_copy_constructible_v<record_type>
+                      || std::is_nothrow_move_constructible_v<record_type>) {
+            try {
+                rebuild_for_capacity(next, size());
+            } catch (...) {
+                // Keep the current (or already committed) capacity/layout and
+                // retry at a later resize boundary.
+            }
+        }
+    }
+
+    void rebuild_for_capacity(size_type target_capacity, size_type target_size) {
+        target_capacity = std::max(target_capacity, target_size);
+        const bool growing = target_capacity >= logical_capacity_;
+        if constexpr (HashIndexEnabled) {
+            if (growing) {
+                index_state_.values.reserve_for_elements(target_capacity);
+                index_state_.ids.reserve(
+                    std::max(index_state_.ids.size(), target_capacity));
+            }
+        }
+        auto target_mode = mode();
+        if (residency_ == ResidencyMode::forced_vector) {
+            target_mode = StorageMode::vector;
+        } else if (residency_ == ResidencyMode::forced_tiered) {
+            target_mode = StorageMode::tiered;
+        } else {
+            target_mode = automatic_mode_for(target_size);
         }
 
-        if constexpr (requires(AdaptationPolicy& p, StorageMode current,
-                               size_type count, TieredConfig config) {
-                          p.recommended_decision(current, count, config);
-                      }) {
-            const auto recommendation =
-                policy_.recommended_decision(mode(), size(), tiered_config_);
-            if (!recommendation) {
-                return false;
-            }
-            if (recommendation->target == mode()) {
-                if (mode() == StorageMode::tiered
-                    && recommendation->tiered_config != tiered_config_) {
-                    reconfigure_tiered(recommendation->tiered_config);
-                    return true;
+        if (target_mode == StorageMode::vector) {
+            vector_storage rebuilt;
+            rebuilt.reserve(target_capacity);
+            if (mode() == StorageMode::vector) {
+                auto& source = std::get<vector_storage>(storage_);
+                if constexpr (std::is_copy_constructible_v<record_type>) {
+                    rebuilt.insert(rebuilt.end(), source.begin(), source.end());
+                } else {
+                    rebuilt.insert(rebuilt.end(),
+                        std::make_move_iterator(source.begin()),
+                        std::make_move_iterator(source.end()));
                 }
-                return false;
+            } else if constexpr (std::is_copy_constructible_v<record_type>) {
+                const auto& source = std::get<tiered_storage>(storage_);
+                source.for_each([&](const record_type& record) {
+                    rebuilt.push_back(record);
+                });
+            } else {
+                auto& source = std::get<tiered_storage>(storage_);
+                source.for_each([&](record_type& record) {
+                    rebuilt.push_back(std::move(record));
+                });
             }
-            if (recommendation->target == StorageMode::tiered) {
-                const auto previous_config = tiered_config_;
-                tiered_config_ = recommendation->tiered_config;
-                try {
-                    convert_to(recommendation->target);
-                } catch (...) {
-                    // The recommended shape is policy state, not committed
-                    // container state.  A failed allocation/copy must not make
-                    // tiered_config() describe storage that was never built.
-                    tiered_config_ = previous_config;
-                    throw;
-                }
-                return true;
+
+            const auto source_mode = mode();
+            storage_.template emplace<vector_storage>(std::move(rebuilt));
+            logical_capacity_ = target_capacity;
+            refresh_vector_locations(0);
+            ++generation_;
+            if (source_mode != StorageMode::vector) {
+                notify_policy_transition(source_mode, StorageMode::vector);
             }
-            convert_to(recommendation->target);
-            return true;
         } else {
-            const auto recommendation = policy_.recommended_mode(mode(), size());
-            if (!recommendation || *recommendation == mode()) {
-                return false;
+            const auto target_config = geometry_for(target_size);
+            tiered_storage rebuilt;
+            if (mode() == StorageMode::vector) {
+                if constexpr (std::is_copy_constructible_v<record_type>) {
+                    rebuilt = tiered_storage::from_vector_copy(
+                        std::get<vector_storage>(storage_), target_config);
+                } else {
+                    rebuilt = tiered_storage::from_vector(
+                        std::move(std::get<vector_storage>(storage_)), target_config);
+                }
+            } else if constexpr (std::is_copy_constructible_v<record_type>) {
+                rebuilt = tiered_storage::reconfigured_copy(
+                    std::get<tiered_storage>(storage_), target_config);
+            } else {
+                rebuilt = tiered_storage::reconfigured_move(
+                    std::get<tiered_storage>(storage_), target_config);
+            }
+
+            const auto source_mode = mode();
+            tiered_config_ = rebuilt.config();
+            storage_.template emplace<tiered_storage>(std::move(rebuilt));
+            logical_capacity_ = target_capacity;
+            refresh_tiered_locations();
+            ++generation_;
+            notify_policy_transition(source_mode, StorageMode::tiered);
+        }
+
+        if constexpr (HashIndexEnabled) {
+            if (!growing) {
+                index_state_.values.reserve_for_elements(target_capacity);
             }
-            convert_to(*recommendation);
-            return true;
         }
     }
 
@@ -911,8 +1244,8 @@ private:
                 tiered = tiered_storage::from_vector_copy(
                     std::get<vector_storage>(storage_), tiered_config_);
             } else {
-                auto values = std::move(std::get<vector_storage>(storage_));
-                tiered = tiered_storage::from_vector(std::move(values), tiered_config_);
+                tiered = tiered_storage::from_vector(
+                    std::move(std::get<vector_storage>(storage_)), tiered_config_);
             }
             tiered_config_ = tiered.config();
             storage_.template emplace<tiered_storage>(std::move(tiered));
@@ -924,6 +1257,9 @@ private:
             } else {
                 values = std::get<tiered_storage>(storage_).to_vector_move();
             }
+            if (logical_capacity_ > values.capacity()) {
+                values.reserve(logical_capacity_);
+            }
             storage_.template emplace<vector_storage>(std::move(values));
             refresh_vector_locations(0);
         }
@@ -955,8 +1291,8 @@ private:
             rebuilt = tiered_storage::reconfigured_copy(
                 std::get<tiered_storage>(storage_), target_config);
         } else {
-            auto values = std::get<tiered_storage>(storage_).to_vector_move();
-            rebuilt = tiered_storage::from_vector(std::move(values), target_config);
+            rebuilt = tiered_storage::reconfigured_move(
+                std::get<tiered_storage>(storage_), target_config);
         }
         tiered_config_ = rebuilt.config();
         storage_.template emplace<tiered_storage>(std::move(rebuilt));
@@ -1028,6 +1364,7 @@ private:
     [[no_unique_address]] Equal equal_{};
     ResidencyMode residency_ = ResidencyMode::automatic;
     ReadAdaptationMode read_adaptation_ = ReadAdaptationMode::deferred;
+    size_type logical_capacity_ = 0;
     std::size_t read_sample_rate_ = 256;
     mutable std::size_t read_sample_countdown_ = 256;
     std::size_t edit_sample_rate_ = 1;

include/universal_container/flat_hash_index.hpp

@@ -6,6 +6,7 @@
 #include <limits>
 #include <optional>
 #include <stdexcept>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
@@ -38,14 +39,39 @@ public:
     }
 
     Entry& ensure(const Key& key) {
-        maybe_grow();
-        const auto [position, found] = find_insert_position(key);
+        // A moved-from std::vector is allowed to be empty while the scalar
+        // counters retain their old values.  Rebuild lazily so a moved-from
+        // index remains reusable and mask() is never evaluated for zero
+        // buckets.
+        if (buckets_.empty()) {
+            rehash(16);
+        }
+
+        auto [position, found] = find_insert_position(key);
+        if (position == npos) {
+            // Normal growth is performed by reserve_for_elements() at a
+            // logical container-capacity change.  Reaching a completely full
+            // table means that invariant was unavailable (most notably after
+            // reusing a moved-from standalone index), so grow only as a
+            // recovery path rather than as a load-factor policy.
+            if (buckets_.size() > std::numeric_limits<std::size_t>::max() / 2) {
+                throw std::length_error("flat hash index capacity overflow");
+            }
+            rehash(buckets_.size() * 2);
+            const auto retry = find_insert_position(key);
+            position = retry.first;
+            found = retry.second;
+            if (position == npos) {
+                throw std::length_error("flat hash index is full");
+            }
+        }
+
         auto& bucket = buckets_[position];
         if (!found) {
+            bucket.key.emplace(key);
             if (bucket.state == State::tombstone) {
                 --tombstones_;
             }
-            bucket.key.emplace(key);
             bucket.entry = {};
             bucket.state = State::occupied;
             ++size_;
@@ -58,27 +84,62 @@ public:
         if (position == npos) {
             return false;
         }
-        auto& bucket = buckets_[position];
-        bucket.key.reset();
-        bucket.entry = {};
-        bucket.state = State::tombstone;
-        --size_;
-        ++tombstones_;
-        if (tombstones_ > buckets_.size() / 4) {
-            rehash(buckets_.size());
+        if constexpr (std::is_nothrow_invocable_v<const Hash&, const Key&>
+                      && std::is_nothrow_move_assignable_v<Bucket>) {
+            erase_and_compact(position);
+        } else {
+            // Computing hashes or moving a key can throw.  Rebuild a complete
+            // replacement without the erased key, then commit with swap, so a
+            // failed erase never opens a hole in the live probe chain.
+            erase_transactional(position);
         }
+        --size_;
         return true;
     }
 
     void clear() {
+        // Allocate before committing so a failed reset leaves the live table
+        // and its duplicate metadata untouched.
+        std::vector<Bucket> replacement(16);
+        buckets_.swap(replacement);
+        size_ = 0;
+        tombstones_ = 0;
+    }
+
+    // Used by the owning container after moving the live table elsewhere.
+    // Keeping zero buckets avoids an allocation in its noexcept move path;
+    // ensure()/reserve_for_elements() restore the minimum table lazily.
+    void reset_after_move() noexcept {
         buckets_.clear();
         size_ = 0;
         tombstones_ = 0;
-        rehash(16);
     }
 
     [[nodiscard]] std::size_t distinct_values() const noexcept { return size_; }
     [[nodiscard]] std::size_t bucket_count() const noexcept { return buckets_.size(); }
+    [[nodiscard]] static std::size_t buckets_for_elements(
+        std::size_t element_capacity) {
+        // At most one distinct key per element.  Keeping the worst-case load
+        // below 70% guarantees that ensure() cannot trigger an independent
+        // rehash between container capacity changes.
+        const auto required = element_capacity > (std::numeric_limits<std::size_t>::max() - 6) / 10
+            ? std::numeric_limits<std::size_t>::max()
+            : (element_capacity * 10 + 6) / 7;
+        std::size_t buckets = 16;
+        while (buckets < required) {
+            if (buckets > std::numeric_limits<std::size_t>::max() / 2) {
+                throw std::length_error("flat hash index capacity overflow");
+            }
+            buckets *= 2;
+        }
+        return buckets;
+    }
+    void reserve_for_elements(std::size_t element_capacity) {
+        const auto requested = buckets_for_elements(element_capacity);
+        if (requested != buckets_.size() || tombstones_ != 0) {
+            rehash(requested);
+        }
+    }
     [[nodiscard]] std::size_t allocated_bytes() const noexcept {
         return buckets_.capacity() * sizeof(Bucket);
     }
@@ -116,10 +177,20 @@ private:
 
     [[nodiscard]] std::pair<std::size_t, bool>
     find_insert_position(const Key& key) const {
-        auto position = hasher_(key) & mask();
+        return find_insert_position_in(buckets_, key);
+    }
+
+    [[nodiscard]] std::pair<std::size_t, bool>
+    find_insert_position_in(const std::vector<Bucket>& buckets,
+                            const Key& key) const {
+        if (buckets.empty()) {
+            return {npos, false};
+        }
+        const auto table_mask = buckets.size() - 1;
+        auto position = hasher_(key) & table_mask;
         auto first_tombstone = npos;
-        for (std::size_t probe = 0; probe < buckets_.size(); ++probe) {
-            const auto& bucket = buckets_[position];
+        for (std::size_t probe = 0; probe < buckets.size(); ++probe) {
+            const auto& bucket = buckets[position];
             if (bucket.state == State::empty) {
                 return {first_tombstone == npos ? position : first_tombstone, false};
             }
@@ -130,41 +201,92 @@ private:
             } else if (equal_(*bucket.key, key)) {
                 return {position, true};
             }
-            position = (position + 1) & mask();
+            position = (position + 1) & table_mask;
         }
         if (first_tombstone != npos) {
             return {first_tombstone, false};
         }
-        throw std::length_error("flat hash index is full");
+        return {npos, false};
     }
 
-    void maybe_grow() {
-        if ((size_ + tombstones_ + 1) * 100 >= buckets_.size() * 82) {
-            rehash(buckets_.size() * 2);
+    void erase_and_compact(std::size_t position) {
+        auto hole = position;
+        buckets_[hole] = Bucket{};
+        auto current = (hole + 1) & mask();
+        while (buckets_[current].state != State::empty) {
+            auto& bucket = buckets_[current];
+            if (bucket.state == State::occupied) {
+                const auto home = hasher_(*bucket.key) & mask();
+                const auto current_distance = (current - home) & mask();
+                const auto hole_distance = (hole - home) & mask();
+                if (hole_distance < current_distance) {
+                    buckets_[hole] = std::move(bucket);
+                    bucket = Bucket{};
+                    hole = current;
+                }
+            }
+            current = (current + 1) & mask();
         }
     }
 
+    void erase_transactional(std::size_t erased_position) {
+        std::vector<Bucket> replacement(buckets_.size());
+        for (std::size_t source = 0; source < buckets_.size(); ++source) {
+            const auto& bucket = buckets_[source];
+            if (source == erased_position || bucket.state != State::occupied) {
+                continue;
+            }
+            const auto [position, found] =
+                find_insert_position_in(replacement, *bucket.key);
+            if (position == npos || found) {
+                throw std::logic_error("flat hash index rebuild invariant violated");
+            }
+            auto& target = replacement[position];
+            target.key.emplace(*bucket.key);
+            target.entry = bucket.entry;
+            target.state = State::occupied;
+        }
+        buckets_.swap(replacement);
+        tombstones_ = 0;
+    }
+
     void rehash(std::size_t requested_capacity) {
         std::size_t capacity = 16;
         while (capacity < requested_capacity) {
+            if (capacity > std::numeric_limits<std::size_t>::max() / 2) {
+                throw std::length_error("flat hash index capacity overflow");
+            }
             capacity *= 2;
         }
-        auto old = std::move(buckets_);
-        buckets_.assign(capacity, Bucket{});
-        size_ = 0;
-        tombstones_ = 0;
-        for (auto& bucket : old) {
+
+        // Build the complete replacement before touching the live table.  An
+        // allocation, hash/equality call, or copy of a copyable Key may throw;
+        // in all of those cases the original buckets and counters remain
+        // unchanged.  vector::swap is the no-throw commit for std::allocator.
+        std::vector<Bucket> replacement(capacity);
+        std::size_t replacement_size = 0;
+        for (const auto& bucket : buckets_) {
             if (bucket.state != State::occupied) {
                 continue;
             }
-            const auto [position, found] = find_insert_position(*bucket.key);
-            (void)found;
-            auto& target = buckets_[position];
-            target.key.emplace(std::move(*bucket.key));
+            const auto [position, found] =
+                find_insert_position_in(replacement, *bucket.key);
+            if (position == npos) {
+                throw std::length_error("flat hash index is full");
+            }
+            if (found) {
+                throw std::logic_error("flat hash index contains duplicate keys");
+            }
+            auto& target = replacement[position];
+            target.key.emplace(*bucket.key);
             target.entry = bucket.entry;
             target.state = State::occupied;
-            ++size_;
+            ++replacement_size;
         }
+
+        buckets_.swap(replacement);
+        size_ = replacement_size;
+        tombstones_ = 0;
     }
 
     Hash hasher_{};

include/universal_container/tiered_storage.hpp

@@ -231,6 +231,31 @@ public:
         }
     }
 
+    template <class Predicate>
+    [[nodiscard]] std::size_t find_if(Predicate&& predicate) const {
+        std::size_t logical = 0;
+        for (const auto& leaf : leaves_) {
+            for (std::size_t i = 0; i < leaf->values.size(); ++i, ++logical) {
+                if (predicate(leaf->values[i])) {
+                    return logical;
+                }
+            }
+        }
+        return npos;
+    }
+
+    template <class Predicate, class Function>
+    void for_each_match(Predicate&& predicate, Function&& function) const {
+        std::size_t logical = 0;
+        for (const auto& leaf : leaves_) {
+            for (std::size_t i = 0; i < leaf->values.size(); ++i, ++logical) {
+                if (predicate(leaf->values[i])) {
+                    function(logical, leaf->values[i]);
+                }
+            }
+        }
+    }
+
     template <class Function>
     void for_each_with_location(Function&& function) const {
         for (const auto& leaf : leaves_) {
@@ -262,21 +287,22 @@ public:
         if (source.empty()) {
             return result;
         }
-        const auto target_occupancy = bulk_target_occupancy(
-            result.config_.leaf_capacity);
-        const auto count = (source.size() + target_occupancy - 1)
-                         / target_occupancy;
-        result.leaves_.reserve(count);
+        // Allocate the complete destination, including its directory, before
+        // moving the first value.  For nothrow-move T the following loop and
+        // the final TieredStorage move cannot fail, so an allocation failure
+        // leaves source completely untouched.
+        const auto target_occupancy = result.prepare_bulk_destination(source.size());
+        std::size_t leaf_index = 0;
         for (auto& value : source) {
-            if (result.leaves_.empty()
-                || result.leaves_.back()->values.size() == target_occupancy) {
-                result.leaves_.push_back(result.make_leaf());
-            }
-            result.leaves_.back()->values.push_back(std::move(value));
+            auto& leaf = *result.leaves_[leaf_index];
+            leaf.values.push_back(std::move(value));
             ++result.size_;
+            if (leaf.values.size() == target_occupancy
+                && leaf_index + 1 < result.leaves_.size()) {
+                ++leaf_index;
+            }
         }
-        result.rebuild_positions();
-        result.rebuild_directory();
+        assert(result.directory_.total() == result.size_);
         for (std::size_t leaf = 0; leaf < result.leaves_.size(); ++leaf) {
             result.refresh_leaf(leaf, relocate);
         }
@@ -335,6 +361,31 @@ public:
         return result;
     }
 
+    // Rebuild a tiered layout without an intermediate vector.  All leaves and
+    // directory arrays are allocated first; consequently this operation has a
+    // strong allocation-failure guarantee when T is nothrow-move-constructible.
+    static TieredStorage reconfigured_move(TieredStorage& source,
+                                            TieredConfig config) {
+        TieredStorage result(config);
+        if (source.empty()) {
+            return result;
+        }
+
+        const auto target_occupancy = result.prepare_bulk_destination(source.size());
+        std::size_t leaf_index = 0;
+        source.for_each([&](T& value) {
+            auto& leaf = *result.leaves_[leaf_index];
+            leaf.values.push_back(std::move(value));
+            ++result.size_;
+            if (leaf.values.size() == target_occupancy
+                && leaf_index + 1 < result.leaves_.size()) {
+                ++leaf_index;
+            }
+        });
+        assert(result.directory_.total() == result.size_);
+        return result;
+    }
+
     [[nodiscard]] std::size_t logical_index(leaf_id_type leaf_id,
                                              std::size_t local) const {
         if (leaf_id >= leaf_positions_.size()) {
@@ -387,6 +438,31 @@ private:
         return std::max<std::size_t>(1, capacity - capacity / 8);
     }
 
+    // Construct the entire shape needed by a bulk load while source elements
+    // are still untouched.  RingBlock allocates all of its optional slots in
+    // its constructor, so inserting into these leaves does not allocate.
+    [[nodiscard]] std::size_t prepare_bulk_destination(std::size_t element_count) {
+        assert(element_count != 0);
+        const auto target_occupancy = bulk_target_occupancy(config_.leaf_capacity);
+        const auto count = 1 + (element_count - 1) / target_occupancy;
+
+        leaves_.reserve(count);
+        leaf_positions_.reserve(count);
+        std::vector<std::size_t> expected_sizes;
+        expected_sizes.reserve(count);
+
+        auto remaining = element_count;
+        for (std::size_t i = 0; i < count; ++i) {
+            leaves_.push_back(make_leaf());
+            const auto leaf_size = std::min(remaining, target_occupancy);
+            expected_sizes.push_back(leaf_size);
+            remaining -= leaf_size;
+        }
+        rebuild_positions();
+        directory_.rebuild(expected_sizes);
+        return target_occupancy;
+    }
+
     [[nodiscard]] std::unique_ptr<Leaf> make_leaf() {
         if (next_leaf_id_ == std::numeric_limits<leaf_id_type>::max()) {
             throw std::length_error("tiered leaf id space exhausted");