Lookich/Polyhedron
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
Polyhedron/tests/TrainingArchiveSelfTest/TrainingArchiveSelfTest.cpp
T
Efim Beshmenev ffd46f89e3 Neural data cache
2026-07-12 14:25:18 +03:00

877 lines
40 KiB
C++
Raw Permalink Blame History

#include "../../src/TrainingArchive/TrainingArchive.h"
#include <array>
#include <cstdint>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
using namespace szilassi::training;
namespace {
constexpr std::uint64_t kTestCap = 10000000;
int fail(int code, const std::string& error = {}) {
std::cerr << "self-test failure " << code;
if (!error.empty()) std::cerr << ": " << error;
std::cerr << '\n';
return code;
}
CommonContext make_context(std::uint64_t sequence = 101) {
CommonContext value;
for (std::size_t index = 0; index < value.run_id.bytes.size(); ++index) {
value.run_id.bytes[index] = static_cast<std::uint8_t>(index + 1);
}
value.record_sequence = sequence;
value.unix_time_ns = 102;
value.topology_fingerprint = 103;
value.algorithm_fingerprint = 104;
value.build_fingerprint = 105;
value.device_fingerprint = 106;
value.run_seed = 107;
value.round_seed = 108;
value.session_seed = 109;
value.session_batch_ordinal = 110;
value.completed_trials = 111;
value.completed_iterations = 112;
value.topology = 42;
value.objective_version = 5;
value.search_mode = 3;
value.flags = 7;
value.batch.chain_count = 16;
value.batch.iterations_per_chain = 200;
value.batch.iterations_per_kernel = 50;
value.batch.shortlist_size = 48;
value.batch.baseline_chains = 4;
value.batch.replica_chains = 2;
value.batch.adaptive_chains = 3;
value.batch.pbt_chains = 1;
value.batch.injected_chains = 6;
value.batch.fresh_numerator = 1;
value.batch.fresh_denominator = 4;
value.batch.replica_group_size = 2;
value.batch.injected_pool_size = 8;
value.batch.evaluated_states = 3200;
value.batch.initialization_evaluated_states = 17;
value.batch.additional_evaluated_states = 23;
value.batch.stagnation_iterations = 150;
value.batch.initial_temperature = 0.25f;
value.batch.final_temperature = 0.01f;
value.batch.proposal_scale = 0.03f;
value.batch.minimum_step = 0.0005f;
value.batch.cooling = 0.995f;
value.batch.minimum_temperature = 0.005f;
value.batch.jump_chance = 0.02f;
value.batch.initial_state_jitter = 0.015f;
value.batch.replica_temperature_ratio = 1.25f;
value.batch.pbt_exploit_chance = 0.18f;
value.batch.pbt_state_jitter = 0.012f;
value.batch.injected_state_jitter = 0.009f;
value.batch.degeneracy_weight = 0.004f;
value.batch.strategy_weights = {{0.1f, 0.2f, 0.25f, 0.15f, 0.3f}};
value.batch.flags = 9;
return value;
}
PlaneState make_state(float base) {
PlaneState value;
for (std::size_t index = 0; index < value.values.size(); ++index) {
value.values[index] = base + static_cast<float>(index) * 0.01f;
}
return value;
}
PrecisePlaneState make_precise_state(double base) {
PrecisePlaneState value;
for (std::size_t index = 0; index < value.values.size(); ++index) {
value.values[index] = base + static_cast<double>(index) * 0.001;
}
return value;
}
Metrics make_metrics(int crossings, int intersections, double base) {
Metrics value;
value.crossings = crossings;
value.intersections = intersections;
value.crossing_loss = base + 0.01;
value.intersection_loss = base + 0.02;
value.geometry_penalty = base + 0.03;
value.degeneracy_penalty = base + 0.04;
value.worst_degeneracy = base + 0.05;
value.energy = base + 0.06;
value.min_plane_determinant = base + 0.07;
value.relative_min_edge = base + 0.08;
value.min_turn_sine = base + 0.09;
value.max_vertex_norm = base + 0.10;
value.condition_number = base + 0.11;
value.condition_penalty = base + 0.12;
value.crossing_face_mask = 0x003;
value.intersection_face_mask = 0x00c;
value.degeneracy_face_mask = 0x030;
value.condition_face_mask = 0x0c0;
value.flags = MetricFinite | MetricCanonical | MetricPrecise | MetricHasSmoothLosses |
MetricHasGeometryPenalty | MetricHasWorstDegeneracy |
MetricHasShapeDescriptors | MetricHasCondition | MetricHasFaceMasks |
MetricHasEnergy;
return value;
}
ApproximateMetrics make_approximate() {
ApproximateMetrics value;
value.crossings = 3;
value.intersections = 4;
value.crossing_loss = 0.31f;
value.intersection_loss = 0.41f;
value.geometry_penalty = 0.51f;
value.degeneracy_penalty = 0.61f;
value.energy = 0.71f;
value.min_plane_determinant = 0.81f;
value.relative_min_edge = 0.91f;
value.min_turn_sine = 0.21f;
value.max_vertex_norm = 1.21f;
value.ambiguity_flags = 0x55;
value.flags = ApproximateHasCounts | ApproximateHasCudaLosses |
ApproximateHasGeometryPenalty | ApproximateHasShapeDescriptors |
ApproximateHasEnergy | ApproximateHasAmbiguity;
return value;
}
VerifiedCandidate make_verified() {
VerifiedCandidate value;
value.context = make_context();
value.backend = BackendSource::CudaShortlist;
value.strategy = 4;
value.chain_index = 7;
value.injected_seed_index = 2;
value.shortlist_rank = 3;
value.chain_seed = 201;
value.chain_iterations = 202;
value.rollout_rng_state = 205;
value.rollout_rng_spare_normal = -0.375f;
value.rollout_rng_has_spare_normal = 1;
value.rollout_start_iterations = 20;
value.origin_batch_ordinal = 12;
value.rollout_start_batch_ordinal = 13;
value.parent_batch_ordinal = 11;
value.best_found_iteration = 180;
value.best_found_batch_ordinal = 17;
value.origin_kind = 3;
value.origin_seed_index = 2;
value.origin_pool_size = 8;
value.origin_seed_replica_count = 2;
value.parent_chain_index = 4;
value.producing_chain_index = 6;
value.reference_hash = 203;
value.candidate_hash = 204;
value.reference_state = make_state(0.1f);
value.reference_metrics = make_metrics(5, 6, 0.2);
value.candidate_state = make_state(0.3f);
value.cuda_metrics = make_approximate();
value.verified_metrics = make_metrics(2, 1, 0.4);
value.flags = CandidateHasReference | CandidateSelectedForCpuVerification |
CandidateImprovedBest | CandidateEnteredArchive | CandidateNeuralGuided;
return value;
}
Fp32Candidate make_fp32_candidate() {
Fp32Candidate value;
value.context = make_context(211);
value.state = make_state(0.42f);
value.approximate_metrics = make_approximate();
value.candidate_hash = 212;
value.chain_initial_rng_state = 213;
value.rollout_rng_state = 214;
value.rollout_rng_spare_normal = 0.625f;
value.rollout_rng_has_spare_normal = 1;
value.chain_iterations = 215;
value.rollout_start_iterations = 31;
value.origin_batch_ordinal = 21;
value.rollout_start_batch_ordinal = 22;
value.parent_batch_ordinal = 20;
value.best_found_iteration = 199;
value.best_found_batch_ordinal = 25;
value.strategy = 3;
value.chain_index = 7;
value.producing_chain_index = 6;
value.injected_seed_index = 1;
value.pool_size = 8;
value.replica_count = 2;
value.origin_kind = 4;
value.origin_seed_index = 3;
value.parent_chain_index = 5;
value.sampling_numerator = 1;
value.sampling_denominator = 8;
value.sampling_probability = 0.125f;
value.flags = 0x41;
return value;
}
SeedProposal make_seed_proposal() {
SeedProposal value;
value.context = make_context(221);
value.purpose = SeedProposalPurpose::BatchInjection;
value.anchor_state = make_state(0.51f);
value.anchor_metrics = make_metrics(7, 6, 0.52);
value.seed_state = make_state(0.53f);
value.seed_metrics = make_metrics(6, 5, 0.54);
for (std::size_t index = 0; index < value.neural_input.size(); ++index) {
value.neural_input[index] = 0.01f + static_cast<float>(index) / 100.0f;
}
value.behavior.improvement_logit = 0.21f;
value.behavior.improvement_probability = 0.61f;
value.behavior.expected_defect_gain = 1.25f;
value.behavior.uncertainty = 0.09f;
value.behavior.plane_probabilities.fill(1.0f / 12.0f);
value.behavior.move_probabilities.fill(0.2f);
value.behavior.scale_probabilities = {{0.25f, 0.5f, 0.25f}};
value.behavior.flags = 3;
value.plane_sampling_distribution.fill(0.05f);
value.plane_sampling_distribution[4] = 0.45f;
value.move_sampling_distribution = {{0.1f, 0.2f, 0.4f, 0.2f, 0.1f}};
value.scale_sampling_distribution = {{0.2f, 0.7f, 0.1f}};
value.plane_action = 4;
value.move_action = 2;
value.scale_action = 1;
value.plane_propensity = 0.45f;
value.move_propensity = 0.4f;
value.scale_propensity = 0.7f;
value.proposal_seed = 222;
value.anchor_hash = 223;
value.seed_hash = 224;
value.seed_index = 3;
value.pool_size = 8;
value.assigned_chains = 2;
value.flags = SeedProposalHasNeuralInput | SeedProposalPredictionSupplied |
SeedProposalNeuralGuided;
return value;
}
InjectedTrajectory make_injected() {
InjectedTrajectory value;
value.context = make_context(301);
value.anchor_state = make_state(0.5f);
value.anchor_metrics = make_metrics(8, 9, 0.6);
value.injected_state = make_state(0.7f);
value.injected_metrics = make_metrics(7, 8, 0.8);
value.result_state = make_state(0.9f);
value.cuda_result_metrics = make_approximate();
value.result_metrics = make_metrics(1, 2, 1.0);
value.anchor_hash = 302;
value.injected_hash = 303;
value.result_hash = 304;
value.rollout_seed = 305;
value.rollout_rng_state = 307;
value.rollout_rng_spare_normal = -0.875f;
value.rollout_rng_has_spare_normal = 1;
value.rollout_iterations = 306;
value.start_iterations = 33;
value.origin_batch_ordinal = 19;
value.injected_seed_index = 5;
value.result_chain_index = 6;
value.pool_size = 8;
value.replica_count = 2;
value.selection_rule = 7;
value.plane_action = 2;
value.move_action = 1;
value.scale_action = 0;
value.plane_sampling_distribution.fill(0.05f);
value.plane_sampling_distribution[2] = 0.45f;
value.move_sampling_distribution = {{0.1f, 0.4f, 0.2f, 0.2f, 0.1f}};
value.scale_sampling_distribution = {{0.6f, 0.3f, 0.1f}};
value.plane_propensity = 0.45f;
value.move_propensity = 0.4f;
value.scale_propensity = 0.6f;
value.behavior.improvement_logit = 0.1f;
value.behavior.improvement_probability = 0.6f;
value.behavior.expected_defect_gain = 0.7f;
value.behavior.uncertainty = 0.08f;
value.behavior.plane_probabilities.fill(1.0f / 12.0f);
value.behavior.move_probabilities.fill(0.2f);
value.behavior.scale_probabilities = {{0.2f, 0.5f, 0.3f}};
value.behavior.flags = 1;
for (std::size_t index = 0; index < value.neural_input.size(); ++index) {
value.neural_input[index] = 0.02f + static_cast<float>(index) / 90.0f;
}
value.flags = TrajectoryNeuralGuided | TrajectoryPredictionSupplied |
TrajectoryImproved | TrajectoryEnteredArchive | TrajectoryHasNeuralInput;
return value;
}
LegacyReplaySample make_legacy(std::uint64_t sequence) {
LegacyReplaySample value;
value.context = make_context(sequence + 400);
for (std::size_t index = 0; index < value.input.size(); ++index) {
value.input[index] = static_cast<float>(index) / 45.0f;
}
value.improved = 1.0f;
value.defect_gain = 2.0f;
value.plane_target[3] = 1.0f;
value.move_target[2] = 1.0f;
value.scale_target[1] = 1.0f;
value.value_weight = 1.2f;
value.plane_weight = 1.3f;
value.move_weight = 1.4f;
value.scale_weight = 1.5f;
value.sequence = sequence;
return value;
}
RefinementTrajectory make_refinement() {
RefinementTrajectory value;
value.context = make_context(501);
value.method = RefinementMethod::Spsa;
value.start_state = make_state(1.1f);
value.start_metrics = make_metrics(4, 5, 1.2);
value.result_state = make_state(1.3f);
value.result_metrics = make_metrics(2, 3, 1.4);
value.start_hash = 502;
value.result_hash = 503;
value.seed = 504;
value.iterations = 505;
value.evaluated_states = 506;
for (std::uint64_t iteration : std::array<std::uint64_t, 2>{{10, 17}}) {
RefinementStep step;
step.iteration = iteration;
step.perturbation = 0.025 + static_cast<double>(iteration) * 0.0001;
step.learning_rate = 0.01 + static_cast<double>(iteration) * 0.0002;
for (std::size_t index = 0; index < step.direction.size(); ++index) {
step.direction[index] = ((index + iteration) & 1U) == 0U ? 1 : -1;
}
step.center_state = make_precise_state(1.5 + iteration * 0.01);
step.plus_state = make_precise_state(1.6 + iteration * 0.01);
step.minus_state = make_precise_state(1.7 + iteration * 0.01);
step.updated_state = make_precise_state(1.8 + iteration * 0.01);
step.plus_metrics = make_metrics(3, 2, 1.9 + iteration * 0.01);
step.minus_metrics = make_metrics(4, 3, 2.0 + iteration * 0.01);
step.flags = RefinementStepPlusEvaluated | RefinementStepMinusEvaluated |
RefinementStepUpdateApplied | RefinementStepImproved;
value.steps.push_back(step);
}
value.flags = RefinementAccepted | RefinementImprovedBest | RefinementEnteredArchive;
return value;
}
std::vector<char> read_bytes(const std::filesystem::path& path) {
std::ifstream input(path, std::ios::binary);
return std::vector<char>(std::istreambuf_iterator<char>(input), {});
}
bool write_bytes(const std::filesystem::path& path, const std::vector<char>& bytes) {
std::ofstream output(path, std::ios::binary | std::ios::trunc);
output.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
return static_cast<bool>(output);
}
bool corrupt_byte(const std::filesystem::path& path, std::uint64_t offset) {
std::fstream file(path, std::ios::binary | std::ios::in | std::ios::out);
file.seekg(static_cast<std::streamoff>(offset));
char value = 0;
file.read(&value, 1);
if (!file) return false;
value ^= 0x5a;
file.seekp(static_cast<std::streamoff>(offset));
file.write(&value, 1);
return static_cast<bool>(file);
}
WriterConfig config_for(const std::filesystem::path& directory, std::uint64_t cap = kTestCap) {
WriterConfig config;
config.shard_directory = directory;
config.global_cap_bytes = cap;
config.target_payload_bytes = 64 * 1024;
return config;
}
} // namespace
int main(int argc, char* argv[]) {
if (argc == 2) {
TrainingShard shard;
std::string inspect_error;
if (!read_shard(argv[1], shard, &inspect_error)) {
return fail(90, inspect_error);
}
std::cout << "records=" << shard.info.record_count
<< " verified=" << shard.verified_candidates.size()
<< " fp32=" << shard.fp32_candidates.size()
<< " seeds=" << shard.seed_proposals.size()
<< " rollouts=" << shard.injected_trajectories.size()
<< " refinements=" << shard.refinement_trajectories.size()
<< " legacy=" << shard.legacy_replay.size() << '\n';
return 0;
}
const std::filesystem::path root = "runtime/training_archive_selftest_data";
std::error_code cleanup_error;
std::filesystem::remove_all(root, cleanup_error);
std::string error;
// Rich schema-v3 round trip and resume of a committed WAL.
WriterConfig rich_config = config_for(root / "rich");
{
TrainingArchiveWriter writer(rich_config);
if (!writer.append(make_verified()).accepted ||
!writer.append(make_fp32_candidate()).accepted ||
!writer.append(make_seed_proposal()).accepted ||
!writer.append(make_injected()).accepted ||
!writer.append(make_legacy(601)).accepted ||
!writer.append(make_refinement()).accepted || !writer.flush(&error)) {
return fail(1, error);
}
}
if (!scan_archive_bytes(root, rich_config.initial_existing_bytes, &error)) return fail(2, error);
{
TrainingArchiveWriter writer(rich_config);
const AppendResult resumed_append = writer.append(make_legacy(602));
if (!resumed_append.accepted) {
std::string notice;
writer.take_write_error_notice(notice);
return fail(3, notice);
}
if (!writer.seal(&error)) {
return fail(3, error);
}
}
TrainingShard rich;
if (!read_shard(root / "rich" / "shard_00000000.sztd", rich, &error)) return fail(4, error);
if (rich.info.record_count != 7 || rich.verified_candidates.size() != 1 ||
rich.fp32_candidates.size() != 1 || rich.seed_proposals.size() != 1 ||
rich.injected_trajectories.size() != 1 || rich.legacy_replay.size() != 2 ||
rich.refinement_trajectories.size() != 1) return fail(5);
const VerifiedCandidate& verified = rich.verified_candidates.front();
const Fp32Candidate& fp32 = rich.fp32_candidates.front();
const SeedProposal& proposal = rich.seed_proposals.front();
const InjectedTrajectory& injected = rich.injected_trajectories.front();
if (verified.context.session_seed != 109 || verified.context.session_batch_ordinal != 110 ||
verified.context.batch.iterations_per_kernel != 50 ||
verified.context.batch.fresh_numerator != 1 ||
verified.context.batch.fresh_denominator != 4 ||
verified.context.batch.replica_group_size != 2 ||
verified.context.batch.injected_pool_size != 8 ||
verified.context.batch.stagnation_iterations != 150 ||
verified.context.batch.minimum_step != 0.0005f ||
verified.context.batch.cooling != 0.995f ||
verified.context.batch.minimum_temperature != 0.005f ||
verified.context.batch.jump_chance != 0.02f ||
verified.context.batch.initial_state_jitter != 0.015f ||
verified.context.batch.replica_temperature_ratio != 1.25f ||
verified.context.batch.pbt_exploit_chance != 0.18f ||
verified.context.batch.pbt_state_jitter != 0.012f ||
verified.context.batch.injected_state_jitter != 0.009f ||
verified.context.batch.strategy_weights[4] != 0.3f || verified.candidate_hash != 204 ||
verified.candidate_state.values[35] != make_state(0.3f).values[35] ||
verified.cuda_metrics.ambiguity_flags != 0x55 ||
verified.rollout_rng_state != 205 ||
verified.rollout_rng_spare_normal != -0.375f ||
verified.rollout_rng_has_spare_normal != 1 ||
verified.rollout_start_iterations != 20 ||
verified.origin_batch_ordinal != 12 || verified.rollout_start_batch_ordinal != 13 ||
verified.parent_batch_ordinal != 11 || verified.best_found_iteration != 180 ||
verified.best_found_batch_ordinal != 17 || verified.origin_kind != 3 ||
verified.origin_seed_index != 2 || verified.origin_pool_size != 8 ||
verified.origin_seed_replica_count != 2 || verified.parent_chain_index != 4 ||
verified.producing_chain_index != 6 ||
fp32.context.record_sequence != 211 || fp32.state.values[35] != make_state(0.42f).values[35] ||
fp32.approximate_metrics.energy != make_approximate().energy ||
fp32.candidate_hash != 212 || fp32.chain_initial_rng_state != 213 ||
fp32.rollout_rng_state != 214 || fp32.rollout_rng_spare_normal != 0.625f ||
fp32.rollout_rng_has_spare_normal != 1 || fp32.chain_iterations != 215 ||
fp32.rollout_start_iterations != 31 || fp32.origin_batch_ordinal != 21 ||
fp32.rollout_start_batch_ordinal != 22 || fp32.parent_batch_ordinal != 20 ||
fp32.best_found_iteration != 199 || fp32.best_found_batch_ordinal != 25 ||
fp32.strategy != 3 || fp32.chain_index != 7 || fp32.producing_chain_index != 6 ||
fp32.injected_seed_index != 1 || fp32.pool_size != 8 || fp32.replica_count != 2 ||
fp32.origin_kind != 4 || fp32.origin_seed_index != 3 ||
fp32.parent_chain_index != 5 || fp32.sampling_numerator != 1 ||
fp32.sampling_denominator != 8 || fp32.sampling_probability != 0.125f ||
fp32.flags != 0x41 ||
proposal.purpose != SeedProposalPurpose::BatchInjection ||
proposal.anchor_state.values[35] != make_state(0.51f).values[35] ||
proposal.anchor_metrics.crossings != 7 ||
proposal.seed_state.values[35] != make_state(0.53f).values[35] ||
proposal.seed_metrics.intersections != 5 || proposal.neural_input[44] == 0.0f ||
proposal.behavior.flags != 3 || proposal.plane_action != 4 ||
proposal.move_action != 2 || proposal.scale_action != 1 ||
proposal.plane_sampling_distribution[4] != 0.45f ||
proposal.move_sampling_distribution[2] != 0.4f ||
proposal.scale_sampling_distribution[1] != 0.7f ||
proposal.plane_propensity != 0.45f || proposal.move_propensity != 0.4f ||
proposal.scale_propensity != 0.7f || proposal.proposal_seed != 222 ||
proposal.anchor_hash != 223 || proposal.seed_hash != 224 ||
proposal.seed_index != 3 || proposal.pool_size != 8 ||
proposal.assigned_chains != 2 ||
proposal.flags != (SeedProposalHasNeuralInput | SeedProposalPredictionSupplied |
SeedProposalNeuralGuided) ||
injected.plane_action != 2 || injected.plane_propensity != 0.45f ||
injected.plane_sampling_distribution[2] != 0.45f ||
injected.rollout_seed != 305 || injected.rollout_rng_state != 307 ||
injected.rollout_rng_spare_normal != -0.875f ||
injected.rollout_rng_has_spare_normal != 1 ||
injected.start_iterations != 33 || injected.origin_batch_ordinal != 19 ||
injected.pool_size != 8 || injected.replica_count != 2 ||
injected.selection_rule != 7 || injected.neural_input[44] == 0.0f ||
(injected.flags & TrajectoryHasNeuralInput) == 0 ||
rich.refinement_trajectories.front().steps.size() != 2 ||
rich.refinement_trajectories.front().steps[0].iteration != 10 ||
rich.refinement_trajectories.front().steps[0].perturbation == 0.0 ||
rich.refinement_trajectories.front().steps[0].learning_rate == 0.0 ||
rich.refinement_trajectories.front().steps[0].direction[0] != 1 ||
rich.refinement_trajectories.front().steps[0].center_state.values[35] == 0.0 ||
rich.refinement_trajectories.front().steps[0].plus_state.values[35] == 0.0 ||
rich.refinement_trajectories.front().steps[0].minus_state.values[35] == 0.0 ||
rich.refinement_trajectories.front().steps[0].updated_state.values[35] == 0.0 ||
rich.refinement_trajectories.front().steps[0].plus_metrics.crossings != 3 ||
rich.refinement_trajectories.front().steps[0].minus_metrics.intersections != 3 ||
rich.refinement_trajectories.front().steps[0].flags !=
(RefinementStepPlusEvaluated | RefinementStepMinusEvaluated |
RefinementStepUpdateApplied | RefinementStepImproved) ||
rich.refinement_trajectories.front().flags !=
(RefinementAccepted | RefinementImprovedBest | RefinementEnteredArchive)) return fail(6);
const std::array<RecordType, 7> expected_order{{
RecordType::VerifiedCandidate,
RecordType::Fp32Candidate,
RecordType::SeedProposal,
RecordType::InjectedTrajectory,
RecordType::LegacyReplay,
RecordType::RefinementTrajectory,
RecordType::LegacyReplay,
}};
if (rich.record_order.size() != expected_order.size()) return fail(42);
for (std::size_t index = 0; index < expected_order.size(); ++index) {
if (rich.record_order[index].type != expected_order[index]) return fail(43);
}
if (rich.record_order[4].index_within_type != 0 ||
rich.record_order[6].index_within_type != 1) return fail(44);
// Decoding and re-encoding every rich record must be byte-identical. This
// catches fields that a partial hand-written round-trip assertion could
// accidentally omit.
{
TrainingArchiveWriter writer(config_for(root / "rich_reencoded"));
if (!writer.append(rich.verified_candidates[0]).accepted ||
!writer.append(rich.fp32_candidates[0]).accepted ||
!writer.append(rich.seed_proposals[0]).accepted ||
!writer.append(rich.injected_trajectories[0]).accepted ||
!writer.append(rich.legacy_replay[0]).accepted ||
!writer.append(rich.refinement_trajectories[0]).accepted ||
!writer.append(rich.legacy_replay[1]).accepted ||
!writer.seal(&error)) {
return fail(27, error);
}
}
if (read_bytes(root / "rich" / "shard_00000000.sztd") !=
read_bytes(root / "rich_reencoded" / "shard_00000000.sztd")) {
return fail(28);
}
// All policy actions must be present together and propensity must match the
// actual normalized distribution used for sampling.
{
TrainingArchiveWriter writer(config_for(root / "invalid_policy"));
InjectedTrajectory partial = make_injected();
partial.move_action = -1;
if (writer.append(partial).status != AppendStatus::InvalidRecord) return fail(7);
InjectedTrajectory mismatch = make_injected();
mismatch.plane_propensity = 0.2f;
if (writer.append(mismatch).status != AppendStatus::InvalidRecord) return fail(8);
VerifiedCandidate bad_ratio = make_verified();
bad_ratio.context.batch.fresh_numerator = 2;
bad_ratio.context.batch.fresh_denominator = 1;
if (writer.append(bad_ratio).status != AppendStatus::InvalidRecord) return fail(45);
VerifiedCandidate bad_evaluation_breakdown = make_verified();
bad_evaluation_breakdown.context.batch.initialization_evaluated_states =
bad_evaluation_breakdown.context.batch.evaluated_states + 1;
if (writer.append(bad_evaluation_breakdown).status != AppendStatus::InvalidRecord) return fail(56);
Fp32Candidate bad_probability = make_fp32_candidate();
bad_probability.sampling_probability = 1.01f;
if (writer.append(bad_probability).status != AppendStatus::InvalidRecord) return fail(46);
Fp32Candidate bad_sampling_ratio = make_fp32_candidate();
bad_sampling_ratio.sampling_denominator = 7;
if (writer.append(bad_sampling_ratio).status != AppendStatus::InvalidRecord) return fail(49);
VerifiedCandidate bad_rng_snapshot = make_verified();
bad_rng_snapshot.rollout_rng_has_spare_normal = 0;
if (writer.append(bad_rng_snapshot).status != AppendStatus::InvalidRecord) return fail(50);
VerifiedCandidate failed_verification = make_verified();
failed_verification.flags = CandidateSelectedForCpuVerification |
CandidateVerificationFailed;
failed_verification.verified_metrics = {};
if (!writer.append(failed_verification).accepted) return fail(51);
failed_verification.flags |= CandidateImprovedBest;
if (writer.append(failed_verification).status != AppendStatus::InvalidRecord) return fail(52);
SeedProposal bad_proposal = make_seed_proposal();
bad_proposal.scale_propensity = 0.2f;
if (writer.append(bad_proposal).status != AppendStatus::InvalidRecord) return fail(47);
RefinementTrajectory bad_refinement = make_refinement();
bad_refinement.steps[0].direction[0] = 0;
if (writer.append(bad_refinement).status != AppendStatus::InvalidRecord) return fail(48);
RefinementTrajectory failed_refinement = make_refinement();
failed_refinement.flags = RefinementVerificationFailed;
failed_refinement.result_state = failed_refinement.start_state;
failed_refinement.result_metrics = failed_refinement.start_metrics;
failed_refinement.result_hash = failed_refinement.start_hash;
if (!writer.append(failed_refinement).accepted) return fail(54);
failed_refinement.flags |= RefinementAccepted;
if (writer.append(failed_refinement).status != AppendStatus::InvalidRecord) return fail(55);
}
// A corrupted complete frame inside the last committed prefix is a hard
// error and must not be silently truncated.
const std::filesystem::path committed_corrupt = root / "committed_corrupt";
{
TrainingArchiveWriter writer(config_for(committed_corrupt));
if (!writer.append(make_legacy(701)).accepted || !writer.flush(&error)) return fail(9, error);
}
if (!corrupt_byte(committed_corrupt / "active.sztd.wal", 48)) return fail(10);
RecoveryReport report;
if (recover_archive_wals(committed_corrupt, kTestCap, report, &error)) return fail(11);
std::filesystem::remove_all(committed_corrupt, cleanup_error);
// Arbitrary corruption in a non-final frame of an uncommitted suffix is
// discarded to the previous durable commit before semantic scanning.
const std::filesystem::path suffix_root = root / "suffix";
std::vector<char> old_commit;
std::uint64_t committed_length = 0;
{
TrainingArchiveWriter writer(config_for(suffix_root));
if (!writer.append(make_legacy(801)).accepted || !writer.flush(&error)) return fail(12, error);
committed_length = std::filesystem::file_size(suffix_root / "active.sztd.wal");
old_commit = read_bytes(suffix_root / "active.sztd.wal.commit");
if (!writer.append(make_legacy(802)).accepted ||
!writer.append(make_legacy(803)).accepted) return fail(13);
} // destructor commits the suffix; restore the old marker to emulate power loss.
if (!write_bytes(suffix_root / "active.sztd.wal.commit", old_commit) ||
!corrupt_byte(suffix_root / "active.sztd.wal", committed_length + 8)) return fail(14);
report = {};
if (!recover_archive_wals(suffix_root, kTestCap, report, &error)) return fail(15, error);
TrainingShard suffix_shard;
if (!read_shard(suffix_root / "shard_00000000.sztd", suffix_shard, &error) ||
suffix_shard.info.record_count != 1 || report.torn_bytes_discarded == 0) return fail(16, error);
// Exact final size is accepted, immediately sealed, and raises one notice.
std::uint64_t pending_one_record = 0;
{
const std::filesystem::path measure = root / "measure";
TrainingArchiveWriter writer(config_for(measure));
if (!writer.append(make_legacy(901)).accepted) return fail(17);
pending_one_record = writer.pending_bytes();
}
std::filesystem::remove_all(root / "measure", cleanup_error);
const std::uint64_t exact_cap = pending_one_record + 40; // format-v3 footer size.
{
TrainingArchiveWriter writer(config_for(root / "exact_cap", exact_cap));
if (!writer.append(make_legacy(902)).accepted || !writer.limit_reached()) return fail(18);
std::string notice;
if (!writer.take_limit_notice(notice) || writer.take_limit_notice(notice) ||
writer.committed_bytes() != exact_cap ||
writer.append(make_legacy(903)).status != AppendStatus::LimitReached) return fail(19);
}
std::uint64_t exact_cap_bytes = 0;
if (!scan_archive_bytes(root / "exact_cap", exact_cap_bytes, &error) ||
exact_cap_bytes != exact_cap) {
return fail(29, error);
}
// A second flush with no new records is a no-op for the commit sidecar.
// Only 48 bytes remain after the first commit: too little for another
// 56-byte temporary marker, but enough to seal the 40-byte footer.
const std::uint64_t noop_flush_cap = pending_one_record + 56 + 48;
{
TrainingArchiveWriter writer(config_for(root / "noop_flush", noop_flush_cap));
if (!writer.append(make_legacy(951)).accepted || !writer.flush(&error) ||
!writer.flush(&error) || !writer.seal(&error) ||
writer.committed_bytes() > noop_flush_cap ||
writer.append(make_legacy(952)).status != AppendStatus::LimitReached) {
return fail(20, error);
}
}
// Atomic commit replacement near a small cap: both dirty flushes and final
// seal succeed without exceeding the cap.
const std::uint64_t replacement_cap = pending_one_record * 3 + 4096;
{
TrainingArchiveWriter writer(config_for(root / "replace_commit", replacement_cap));
if (!writer.append(make_legacy(1001)).accepted || !writer.flush(&error) ||
!writer.append(make_legacy(1002)).accepted || !writer.flush(&error) ||
!writer.seal(&error) || writer.committed_bytes() > replacement_cap) {
return fail(21, error);
}
}
// Commit artifacts without a WAL contain no records. Recovery removes both
// the official marker and a crashed atomic-replacement temporary.
const std::filesystem::path stale = root / "stale";
std::filesystem::create_directories(stale);
if (!write_bytes(stale / "active.sztd.wal.commit", {'s', 't', 'a', 'l', 'e'}) ||
!write_bytes(stale / "active.sztd.wal.commit.tmp", {'t', 'm', 'p'})) return fail(22);
report = {};
if (!recover_archive_wals(stale, kTestCap, report, &error) ||
std::filesystem::exists(stale / "active.sztd.wal.commit") ||
std::filesystem::exists(stale / "active.sztd.wal.commit.tmp")) return fail(23, error);
// A conflicting destination is never replaced or modified.
const std::filesystem::path collision = root / "collision";
{
TrainingArchiveWriter writer(config_for(collision));
if (!writer.append(make_legacy(1101)).accepted || !writer.flush(&error)) return fail(24, error);
const std::vector<char> sentinel{'n', 'o', 't', '-', 'a', '-', 's', 'h', 'a', 'r', 'd'};
if (!write_bytes(collision / "shard_00000000.sztd", sentinel) || writer.seal(&error)) {
return fail(25, error);
}
if (read_bytes(collision / "shard_00000000.sztd") != sentinel) return fail(26);
}
// A crash while the very first uncommitted fixed-size header is being
// written may leave 0..40 garbage bytes. No committed record can exist in
// that state, so recovery removes it instead of permanently wedging start.
const std::filesystem::path partial_headers = root / "partial_headers";
for (std::size_t size : std::array<std::size_t, 4>{{0, 1, 39, 40}}) {
const std::filesystem::path directory =
partial_headers / ("bytes_" + std::to_string(size));
std::filesystem::create_directories(directory);
if (!write_bytes(
directory / "active.sztd.wal",
std::vector<char>(size, static_cast<char>(0x5a)))) {
return fail(30);
}
}
report = {};
if (!recover_archive_wals(partial_headers, kTestCap, report, &error) ||
report.empty_wals_removed != 4) {
return fail(31, error);
}
std::uint64_t partial_bytes = 1;
if (!scan_archive_bytes(partial_headers, partial_bytes, &error) || partial_bytes != 0) {
return fail(32, error);
}
// A valid commit marker makes even header corruption a hard error. The
// recovery attempt must leave both artifacts byte-for-byte untouched.
const std::filesystem::path committed_header = root / "committed_header";
{
TrainingArchiveWriter writer(config_for(committed_header));
if (!writer.append(make_legacy(1201)).accepted || !writer.flush(&error)) {
return fail(33, error);
}
}
const std::filesystem::path committed_header_wal =
committed_header / "active.sztd.wal";
const std::filesystem::path committed_header_marker =
committed_header / "active.sztd.wal.commit";
if (!corrupt_byte(committed_header_wal, 0)) return fail(34);
const std::vector<char> corrupted_header_bytes = read_bytes(committed_header_wal);
const std::vector<char> committed_header_bytes = read_bytes(committed_header_marker);
report = {};
if (recover_archive_wals(committed_header, kTestCap, report, &error) ||
read_bytes(committed_header_wal) != corrupted_header_bytes ||
read_bytes(committed_header_marker) != committed_header_bytes) {
return fail(35, error);
}
// A fully committed WAL can exactly consume the remaining allocation
// while still lacking its 40-byte immutable footer. This is a normal
// cache-limit boundary: recovery must preserve the WAL and marker, and a
// writer opened against the same accounting must freeze collection and
// emit one limit notice rather than reporting an I/O failure.
const std::filesystem::path deferred_at_limit = root / "deferred_at_limit";
{
TrainingArchiveWriter writer(config_for(deferred_at_limit));
if (!writer.append(make_legacy(1251)).accepted || !writer.flush(&error)) {
return fail(49, error);
}
}
const std::filesystem::path deferred_wal =
deferred_at_limit / "active.sztd.wal";
const std::filesystem::path deferred_commit =
deferred_at_limit / "active.sztd.wal.commit";
const std::vector<char> deferred_wal_bytes = read_bytes(deferred_wal);
const std::vector<char> deferred_commit_bytes = read_bytes(deferred_commit);
std::uint64_t deferred_cap = 0;
if (deferred_wal_bytes.empty() || deferred_commit_bytes.empty() ||
!scan_archive_bytes(deferred_at_limit, deferred_cap, &error)) {
return fail(50, error);
}
report = {};
if (!recover_archive_wals(deferred_at_limit, deferred_cap, report, &error) ||
report.wals_found != 1 || report.wals_deferred_at_limit != 1 ||
report.shards_sealed != 0 || report.records_recovered != 0 ||
read_bytes(deferred_wal) != deferred_wal_bytes ||
read_bytes(deferred_commit) != deferred_commit_bytes ||
std::filesystem::exists(
deferred_at_limit / "shard_00000000.sztd")) {
return fail(51, error);
}
WriterConfig deferred_config = config_for(deferred_at_limit, deferred_cap);
deferred_config.initial_existing_bytes = deferred_cap;
{
TrainingArchiveWriter writer(deferred_config);
std::string notice;
if (!writer.limit_reached() || writer.collection_enabled() ||
writer.has_write_error() || !writer.take_limit_notice(notice) ||
notice.empty() || writer.take_limit_notice(notice) ||
writer.append(make_legacy(1252)).status != AppendStatus::LimitReached ||
!writer.flush(&error)) {
return fail(52, error);
}
}
if (read_bytes(deferred_wal) != deferred_wal_bytes ||
read_bytes(deferred_commit) != deferred_commit_bytes) {
return fail(53);
}
// If an equivalent immutable shard already exists, the startup byte scan
// counted both it and the leftover sealed WAL. Recount after deduplication
// so a following committed WAL can use the room that was actually freed.
const std::filesystem::path equivalent_root = root / "equivalent_recovery";
const std::filesystem::path duplicate_dir = equivalent_root / "a_duplicate";
const std::filesystem::path pending_dir = equivalent_root / "z_pending";
{
TrainingArchiveWriter writer(config_for(duplicate_dir));
if (!writer.append(make_legacy(1301)).accepted || !writer.seal(&error)) {
return fail(36, error);
}
}
const std::filesystem::path duplicate_shard =
duplicate_dir / "shard_00000000.sztd";
const std::uint64_t duplicate_size = std::filesystem::file_size(duplicate_shard);
if (!std::filesystem::copy_file(
duplicate_shard,
duplicate_dir / "active.sztd.wal",
std::filesystem::copy_options::none,
cleanup_error) || cleanup_error) {
return fail(37, cleanup_error.message());
}
{
TrainingArchiveWriter writer(config_for(pending_dir));
if (!writer.append(make_legacy(1302)).accepted || !writer.flush(&error)) {
return fail(38, error);
}
}
std::uint64_t equivalent_initial_bytes = 0;
if (!scan_archive_bytes(equivalent_root, equivalent_initial_bytes, &error) ||
equivalent_initial_bytes < duplicate_size) {
return fail(39, error);
}
const std::uint64_t recovery_cap =
equivalent_initial_bytes - duplicate_size + 40; // one v3 footer
report = {};
if (!recover_archive_wals(equivalent_root, recovery_cap, report, &error) ||
std::filesystem::exists(duplicate_dir / "active.sztd.wal") ||
std::filesystem::exists(pending_dir / "active.sztd.wal") ||
std::filesystem::exists(pending_dir / "active.sztd.wal.commit") ||
!std::filesystem::exists(pending_dir / "shard_00000000.sztd")) {
return fail(40, error);
}
std::uint64_t equivalent_final_bytes = 0;
if (!scan_archive_bytes(equivalent_root, equivalent_final_bytes, &error) ||
equivalent_final_bytes > recovery_cap) {
return fail(41, error);
}
std::filesystem::remove_all(root, cleanup_error);
std::cout << "training archive self-test passed\n";
return 0;
}
Reference in New Issue View Git Blame Copy Permalink