Neural data cache
This commit is contained in:
Efim Beshmenev
@@ -0,0 +1,876 @@
|
||||
#include "../../src/TrainingArchive/TrainingArchive.h"
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using namespace szilassi::training;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::uint64_t kTestCap = 10000000;
|
||||
|
||||
int fail(int code, const std::string& error = {}) {
|
||||
std::cerr << "self-test failure " << code;
|
||||
if (!error.empty()) std::cerr << ": " << error;
|
||||
std::cerr << '\n';
|
||||
return code;
|
||||
}
|
||||
|
||||
CommonContext make_context(std::uint64_t sequence = 101) {
|
||||
CommonContext value;
|
||||
for (std::size_t index = 0; index < value.run_id.bytes.size(); ++index) {
|
||||
value.run_id.bytes[index] = static_cast<std::uint8_t>(index + 1);
|
||||
}
|
||||
value.record_sequence = sequence;
|
||||
value.unix_time_ns = 102;
|
||||
value.topology_fingerprint = 103;
|
||||
value.algorithm_fingerprint = 104;
|
||||
value.build_fingerprint = 105;
|
||||
value.device_fingerprint = 106;
|
||||
value.run_seed = 107;
|
||||
value.round_seed = 108;
|
||||
value.session_seed = 109;
|
||||
value.session_batch_ordinal = 110;
|
||||
value.completed_trials = 111;
|
||||
value.completed_iterations = 112;
|
||||
value.topology = 42;
|
||||
value.objective_version = 5;
|
||||
value.search_mode = 3;
|
||||
value.flags = 7;
|
||||
value.batch.chain_count = 16;
|
||||
value.batch.iterations_per_chain = 200;
|
||||
value.batch.iterations_per_kernel = 50;
|
||||
value.batch.shortlist_size = 48;
|
||||
value.batch.baseline_chains = 4;
|
||||
value.batch.replica_chains = 2;
|
||||
value.batch.adaptive_chains = 3;
|
||||
value.batch.pbt_chains = 1;
|
||||
value.batch.injected_chains = 6;
|
||||
value.batch.fresh_numerator = 1;
|
||||
value.batch.fresh_denominator = 4;
|
||||
value.batch.replica_group_size = 2;
|
||||
value.batch.injected_pool_size = 8;
|
||||
value.batch.evaluated_states = 3200;
|
||||
value.batch.initialization_evaluated_states = 17;
|
||||
value.batch.additional_evaluated_states = 23;
|
||||
value.batch.stagnation_iterations = 150;
|
||||
value.batch.initial_temperature = 0.25f;
|
||||
value.batch.final_temperature = 0.01f;
|
||||
value.batch.proposal_scale = 0.03f;
|
||||
value.batch.minimum_step = 0.0005f;
|
||||
value.batch.cooling = 0.995f;
|
||||
value.batch.minimum_temperature = 0.005f;
|
||||
value.batch.jump_chance = 0.02f;
|
||||
value.batch.initial_state_jitter = 0.015f;
|
||||
value.batch.replica_temperature_ratio = 1.25f;
|
||||
value.batch.pbt_exploit_chance = 0.18f;
|
||||
value.batch.pbt_state_jitter = 0.012f;
|
||||
value.batch.injected_state_jitter = 0.009f;
|
||||
value.batch.degeneracy_weight = 0.004f;
|
||||
value.batch.strategy_weights = {{0.1f, 0.2f, 0.25f, 0.15f, 0.3f}};
|
||||
value.batch.flags = 9;
|
||||
return value;
|
||||
}
|
||||
|
||||
PlaneState make_state(float base) {
|
||||
PlaneState value;
|
||||
for (std::size_t index = 0; index < value.values.size(); ++index) {
|
||||
value.values[index] = base + static_cast<float>(index) * 0.01f;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
PrecisePlaneState make_precise_state(double base) {
|
||||
PrecisePlaneState value;
|
||||
for (std::size_t index = 0; index < value.values.size(); ++index) {
|
||||
value.values[index] = base + static_cast<double>(index) * 0.001;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
Metrics make_metrics(int crossings, int intersections, double base) {
|
||||
Metrics value;
|
||||
value.crossings = crossings;
|
||||
value.intersections = intersections;
|
||||
value.crossing_loss = base + 0.01;
|
||||
value.intersection_loss = base + 0.02;
|
||||
value.geometry_penalty = base + 0.03;
|
||||
value.degeneracy_penalty = base + 0.04;
|
||||
value.worst_degeneracy = base + 0.05;
|
||||
value.energy = base + 0.06;
|
||||
value.min_plane_determinant = base + 0.07;
|
||||
value.relative_min_edge = base + 0.08;
|
||||
value.min_turn_sine = base + 0.09;
|
||||
value.max_vertex_norm = base + 0.10;
|
||||
value.condition_number = base + 0.11;
|
||||
value.condition_penalty = base + 0.12;
|
||||
value.crossing_face_mask = 0x003;
|
||||
value.intersection_face_mask = 0x00c;
|
||||
value.degeneracy_face_mask = 0x030;
|
||||
value.condition_face_mask = 0x0c0;
|
||||
value.flags = MetricFinite | MetricCanonical | MetricPrecise | MetricHasSmoothLosses |
|
||||
MetricHasGeometryPenalty | MetricHasWorstDegeneracy |
|
||||
MetricHasShapeDescriptors | MetricHasCondition | MetricHasFaceMasks |
|
||||
MetricHasEnergy;
|
||||
return value;
|
||||
}
|
||||
|
||||
ApproximateMetrics make_approximate() {
|
||||
ApproximateMetrics value;
|
||||
value.crossings = 3;
|
||||
value.intersections = 4;
|
||||
value.crossing_loss = 0.31f;
|
||||
value.intersection_loss = 0.41f;
|
||||
value.geometry_penalty = 0.51f;
|
||||
value.degeneracy_penalty = 0.61f;
|
||||
value.energy = 0.71f;
|
||||
value.min_plane_determinant = 0.81f;
|
||||
value.relative_min_edge = 0.91f;
|
||||
value.min_turn_sine = 0.21f;
|
||||
value.max_vertex_norm = 1.21f;
|
||||
value.ambiguity_flags = 0x55;
|
||||
value.flags = ApproximateHasCounts | ApproximateHasCudaLosses |
|
||||
ApproximateHasGeometryPenalty | ApproximateHasShapeDescriptors |
|
||||
ApproximateHasEnergy | ApproximateHasAmbiguity;
|
||||
return value;
|
||||
}
|
||||
|
||||
VerifiedCandidate make_verified() {
|
||||
VerifiedCandidate value;
|
||||
value.context = make_context();
|
||||
value.backend = BackendSource::CudaShortlist;
|
||||
value.strategy = 4;
|
||||
value.chain_index = 7;
|
||||
value.injected_seed_index = 2;
|
||||
value.shortlist_rank = 3;
|
||||
value.chain_seed = 201;
|
||||
value.chain_iterations = 202;
|
||||
value.rollout_rng_state = 205;
|
||||
value.rollout_rng_spare_normal = -0.375f;
|
||||
value.rollout_rng_has_spare_normal = 1;
|
||||
value.rollout_start_iterations = 20;
|
||||
value.origin_batch_ordinal = 12;
|
||||
value.rollout_start_batch_ordinal = 13;
|
||||
value.parent_batch_ordinal = 11;
|
||||
value.best_found_iteration = 180;
|
||||
value.best_found_batch_ordinal = 17;
|
||||
value.origin_kind = 3;
|
||||
value.origin_seed_index = 2;
|
||||
value.origin_pool_size = 8;
|
||||
value.origin_seed_replica_count = 2;
|
||||
value.parent_chain_index = 4;
|
||||
value.producing_chain_index = 6;
|
||||
value.reference_hash = 203;
|
||||
value.candidate_hash = 204;
|
||||
value.reference_state = make_state(0.1f);
|
||||
value.reference_metrics = make_metrics(5, 6, 0.2);
|
||||
value.candidate_state = make_state(0.3f);
|
||||
value.cuda_metrics = make_approximate();
|
||||
value.verified_metrics = make_metrics(2, 1, 0.4);
|
||||
value.flags = CandidateHasReference | CandidateSelectedForCpuVerification |
|
||||
CandidateImprovedBest | CandidateEnteredArchive | CandidateNeuralGuided;
|
||||
return value;
|
||||
}
|
||||
|
||||
Fp32Candidate make_fp32_candidate() {
|
||||
Fp32Candidate value;
|
||||
value.context = make_context(211);
|
||||
value.state = make_state(0.42f);
|
||||
value.approximate_metrics = make_approximate();
|
||||
value.candidate_hash = 212;
|
||||
value.chain_initial_rng_state = 213;
|
||||
value.rollout_rng_state = 214;
|
||||
value.rollout_rng_spare_normal = 0.625f;
|
||||
value.rollout_rng_has_spare_normal = 1;
|
||||
value.chain_iterations = 215;
|
||||
value.rollout_start_iterations = 31;
|
||||
value.origin_batch_ordinal = 21;
|
||||
value.rollout_start_batch_ordinal = 22;
|
||||
value.parent_batch_ordinal = 20;
|
||||
value.best_found_iteration = 199;
|
||||
value.best_found_batch_ordinal = 25;
|
||||
value.strategy = 3;
|
||||
value.chain_index = 7;
|
||||
value.producing_chain_index = 6;
|
||||
value.injected_seed_index = 1;
|
||||
value.pool_size = 8;
|
||||
value.replica_count = 2;
|
||||
value.origin_kind = 4;
|
||||
value.origin_seed_index = 3;
|
||||
value.parent_chain_index = 5;
|
||||
value.sampling_numerator = 1;
|
||||
value.sampling_denominator = 8;
|
||||
value.sampling_probability = 0.125f;
|
||||
value.flags = 0x41;
|
||||
return value;
|
||||
}
|
||||
|
||||
SeedProposal make_seed_proposal() {
|
||||
SeedProposal value;
|
||||
value.context = make_context(221);
|
||||
value.purpose = SeedProposalPurpose::BatchInjection;
|
||||
value.anchor_state = make_state(0.51f);
|
||||
value.anchor_metrics = make_metrics(7, 6, 0.52);
|
||||
value.seed_state = make_state(0.53f);
|
||||
value.seed_metrics = make_metrics(6, 5, 0.54);
|
||||
for (std::size_t index = 0; index < value.neural_input.size(); ++index) {
|
||||
value.neural_input[index] = 0.01f + static_cast<float>(index) / 100.0f;
|
||||
}
|
||||
value.behavior.improvement_logit = 0.21f;
|
||||
value.behavior.improvement_probability = 0.61f;
|
||||
value.behavior.expected_defect_gain = 1.25f;
|
||||
value.behavior.uncertainty = 0.09f;
|
||||
value.behavior.plane_probabilities.fill(1.0f / 12.0f);
|
||||
value.behavior.move_probabilities.fill(0.2f);
|
||||
value.behavior.scale_probabilities = {{0.25f, 0.5f, 0.25f}};
|
||||
value.behavior.flags = 3;
|
||||
value.plane_sampling_distribution.fill(0.05f);
|
||||
value.plane_sampling_distribution[4] = 0.45f;
|
||||
value.move_sampling_distribution = {{0.1f, 0.2f, 0.4f, 0.2f, 0.1f}};
|
||||
value.scale_sampling_distribution = {{0.2f, 0.7f, 0.1f}};
|
||||
value.plane_action = 4;
|
||||
value.move_action = 2;
|
||||
value.scale_action = 1;
|
||||
value.plane_propensity = 0.45f;
|
||||
value.move_propensity = 0.4f;
|
||||
value.scale_propensity = 0.7f;
|
||||
value.proposal_seed = 222;
|
||||
value.anchor_hash = 223;
|
||||
value.seed_hash = 224;
|
||||
value.seed_index = 3;
|
||||
value.pool_size = 8;
|
||||
value.assigned_chains = 2;
|
||||
value.flags = SeedProposalHasNeuralInput | SeedProposalPredictionSupplied |
|
||||
SeedProposalNeuralGuided;
|
||||
return value;
|
||||
}
|
||||
|
||||
InjectedTrajectory make_injected() {
|
||||
InjectedTrajectory value;
|
||||
value.context = make_context(301);
|
||||
value.anchor_state = make_state(0.5f);
|
||||
value.anchor_metrics = make_metrics(8, 9, 0.6);
|
||||
value.injected_state = make_state(0.7f);
|
||||
value.injected_metrics = make_metrics(7, 8, 0.8);
|
||||
value.result_state = make_state(0.9f);
|
||||
value.cuda_result_metrics = make_approximate();
|
||||
value.result_metrics = make_metrics(1, 2, 1.0);
|
||||
value.anchor_hash = 302;
|
||||
value.injected_hash = 303;
|
||||
value.result_hash = 304;
|
||||
value.rollout_seed = 305;
|
||||
value.rollout_rng_state = 307;
|
||||
value.rollout_rng_spare_normal = -0.875f;
|
||||
value.rollout_rng_has_spare_normal = 1;
|
||||
value.rollout_iterations = 306;
|
||||
value.start_iterations = 33;
|
||||
value.origin_batch_ordinal = 19;
|
||||
value.injected_seed_index = 5;
|
||||
value.result_chain_index = 6;
|
||||
value.pool_size = 8;
|
||||
value.replica_count = 2;
|
||||
value.selection_rule = 7;
|
||||
value.plane_action = 2;
|
||||
value.move_action = 1;
|
||||
value.scale_action = 0;
|
||||
value.plane_sampling_distribution.fill(0.05f);
|
||||
value.plane_sampling_distribution[2] = 0.45f;
|
||||
value.move_sampling_distribution = {{0.1f, 0.4f, 0.2f, 0.2f, 0.1f}};
|
||||
value.scale_sampling_distribution = {{0.6f, 0.3f, 0.1f}};
|
||||
value.plane_propensity = 0.45f;
|
||||
value.move_propensity = 0.4f;
|
||||
value.scale_propensity = 0.6f;
|
||||
value.behavior.improvement_logit = 0.1f;
|
||||
value.behavior.improvement_probability = 0.6f;
|
||||
value.behavior.expected_defect_gain = 0.7f;
|
||||
value.behavior.uncertainty = 0.08f;
|
||||
value.behavior.plane_probabilities.fill(1.0f / 12.0f);
|
||||
value.behavior.move_probabilities.fill(0.2f);
|
||||
value.behavior.scale_probabilities = {{0.2f, 0.5f, 0.3f}};
|
||||
value.behavior.flags = 1;
|
||||
for (std::size_t index = 0; index < value.neural_input.size(); ++index) {
|
||||
value.neural_input[index] = 0.02f + static_cast<float>(index) / 90.0f;
|
||||
}
|
||||
value.flags = TrajectoryNeuralGuided | TrajectoryPredictionSupplied |
|
||||
TrajectoryImproved | TrajectoryEnteredArchive | TrajectoryHasNeuralInput;
|
||||
return value;
|
||||
}
|
||||
|
||||
LegacyReplaySample make_legacy(std::uint64_t sequence) {
|
||||
LegacyReplaySample value;
|
||||
value.context = make_context(sequence + 400);
|
||||
for (std::size_t index = 0; index < value.input.size(); ++index) {
|
||||
value.input[index] = static_cast<float>(index) / 45.0f;
|
||||
}
|
||||
value.improved = 1.0f;
|
||||
value.defect_gain = 2.0f;
|
||||
value.plane_target[3] = 1.0f;
|
||||
value.move_target[2] = 1.0f;
|
||||
value.scale_target[1] = 1.0f;
|
||||
value.value_weight = 1.2f;
|
||||
value.plane_weight = 1.3f;
|
||||
value.move_weight = 1.4f;
|
||||
value.scale_weight = 1.5f;
|
||||
value.sequence = sequence;
|
||||
return value;
|
||||
}
|
||||
|
||||
RefinementTrajectory make_refinement() {
|
||||
RefinementTrajectory value;
|
||||
value.context = make_context(501);
|
||||
value.method = RefinementMethod::Spsa;
|
||||
value.start_state = make_state(1.1f);
|
||||
value.start_metrics = make_metrics(4, 5, 1.2);
|
||||
value.result_state = make_state(1.3f);
|
||||
value.result_metrics = make_metrics(2, 3, 1.4);
|
||||
value.start_hash = 502;
|
||||
value.result_hash = 503;
|
||||
value.seed = 504;
|
||||
value.iterations = 505;
|
||||
value.evaluated_states = 506;
|
||||
for (std::uint64_t iteration : std::array<std::uint64_t, 2>{{10, 17}}) {
|
||||
RefinementStep step;
|
||||
step.iteration = iteration;
|
||||
step.perturbation = 0.025 + static_cast<double>(iteration) * 0.0001;
|
||||
step.learning_rate = 0.01 + static_cast<double>(iteration) * 0.0002;
|
||||
for (std::size_t index = 0; index < step.direction.size(); ++index) {
|
||||
step.direction[index] = ((index + iteration) & 1U) == 0U ? 1 : -1;
|
||||
}
|
||||
step.center_state = make_precise_state(1.5 + iteration * 0.01);
|
||||
step.plus_state = make_precise_state(1.6 + iteration * 0.01);
|
||||
step.minus_state = make_precise_state(1.7 + iteration * 0.01);
|
||||
step.updated_state = make_precise_state(1.8 + iteration * 0.01);
|
||||
step.plus_metrics = make_metrics(3, 2, 1.9 + iteration * 0.01);
|
||||
step.minus_metrics = make_metrics(4, 3, 2.0 + iteration * 0.01);
|
||||
step.flags = RefinementStepPlusEvaluated | RefinementStepMinusEvaluated |
|
||||
RefinementStepUpdateApplied | RefinementStepImproved;
|
||||
value.steps.push_back(step);
|
||||
}
|
||||
value.flags = RefinementAccepted | RefinementImprovedBest | RefinementEnteredArchive;
|
||||
return value;
|
||||
}
|
||||
|
||||
std::vector<char> read_bytes(const std::filesystem::path& path) {
|
||||
std::ifstream input(path, std::ios::binary);
|
||||
return std::vector<char>(std::istreambuf_iterator<char>(input), {});
|
||||
}
|
||||
|
||||
bool write_bytes(const std::filesystem::path& path, const std::vector<char>& bytes) {
|
||||
std::ofstream output(path, std::ios::binary | std::ios::trunc);
|
||||
output.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
|
||||
return static_cast<bool>(output);
|
||||
}
|
||||
|
||||
bool corrupt_byte(const std::filesystem::path& path, std::uint64_t offset) {
|
||||
std::fstream file(path, std::ios::binary | std::ios::in | std::ios::out);
|
||||
file.seekg(static_cast<std::streamoff>(offset));
|
||||
char value = 0;
|
||||
file.read(&value, 1);
|
||||
if (!file) return false;
|
||||
value ^= 0x5a;
|
||||
file.seekp(static_cast<std::streamoff>(offset));
|
||||
file.write(&value, 1);
|
||||
return static_cast<bool>(file);
|
||||
}
|
||||
|
||||
WriterConfig config_for(const std::filesystem::path& directory, std::uint64_t cap = kTestCap) {
|
||||
WriterConfig config;
|
||||
config.shard_directory = directory;
|
||||
config.global_cap_bytes = cap;
|
||||
config.target_payload_bytes = 64 * 1024;
|
||||
return config;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc == 2) {
|
||||
TrainingShard shard;
|
||||
std::string inspect_error;
|
||||
if (!read_shard(argv[1], shard, &inspect_error)) {
|
||||
return fail(90, inspect_error);
|
||||
}
|
||||
std::cout << "records=" << shard.info.record_count
|
||||
<< " verified=" << shard.verified_candidates.size()
|
||||
<< " fp32=" << shard.fp32_candidates.size()
|
||||
<< " seeds=" << shard.seed_proposals.size()
|
||||
<< " rollouts=" << shard.injected_trajectories.size()
|
||||
<< " refinements=" << shard.refinement_trajectories.size()
|
||||
<< " legacy=" << shard.legacy_replay.size() << '\n';
|
||||
return 0;
|
||||
}
|
||||
const std::filesystem::path root = "runtime/training_archive_selftest_data";
|
||||
std::error_code cleanup_error;
|
||||
std::filesystem::remove_all(root, cleanup_error);
|
||||
std::string error;
|
||||
|
||||
// Rich schema-v3 round trip and resume of a committed WAL.
|
||||
WriterConfig rich_config = config_for(root / "rich");
|
||||
{
|
||||
TrainingArchiveWriter writer(rich_config);
|
||||
if (!writer.append(make_verified()).accepted ||
|
||||
!writer.append(make_fp32_candidate()).accepted ||
|
||||
!writer.append(make_seed_proposal()).accepted ||
|
||||
!writer.append(make_injected()).accepted ||
|
||||
!writer.append(make_legacy(601)).accepted ||
|
||||
!writer.append(make_refinement()).accepted || !writer.flush(&error)) {
|
||||
return fail(1, error);
|
||||
}
|
||||
}
|
||||
if (!scan_archive_bytes(root, rich_config.initial_existing_bytes, &error)) return fail(2, error);
|
||||
{
|
||||
TrainingArchiveWriter writer(rich_config);
|
||||
const AppendResult resumed_append = writer.append(make_legacy(602));
|
||||
if (!resumed_append.accepted) {
|
||||
std::string notice;
|
||||
writer.take_write_error_notice(notice);
|
||||
return fail(3, notice);
|
||||
}
|
||||
if (!writer.seal(&error)) {
|
||||
return fail(3, error);
|
||||
}
|
||||
}
|
||||
TrainingShard rich;
|
||||
if (!read_shard(root / "rich" / "shard_00000000.sztd", rich, &error)) return fail(4, error);
|
||||
if (rich.info.record_count != 7 || rich.verified_candidates.size() != 1 ||
|
||||
rich.fp32_candidates.size() != 1 || rich.seed_proposals.size() != 1 ||
|
||||
rich.injected_trajectories.size() != 1 || rich.legacy_replay.size() != 2 ||
|
||||
rich.refinement_trajectories.size() != 1) return fail(5);
|
||||
const VerifiedCandidate& verified = rich.verified_candidates.front();
|
||||
const Fp32Candidate& fp32 = rich.fp32_candidates.front();
|
||||
const SeedProposal& proposal = rich.seed_proposals.front();
|
||||
const InjectedTrajectory& injected = rich.injected_trajectories.front();
|
||||
if (verified.context.session_seed != 109 || verified.context.session_batch_ordinal != 110 ||
|
||||
verified.context.batch.iterations_per_kernel != 50 ||
|
||||
verified.context.batch.fresh_numerator != 1 ||
|
||||
verified.context.batch.fresh_denominator != 4 ||
|
||||
verified.context.batch.replica_group_size != 2 ||
|
||||
verified.context.batch.injected_pool_size != 8 ||
|
||||
verified.context.batch.stagnation_iterations != 150 ||
|
||||
verified.context.batch.minimum_step != 0.0005f ||
|
||||
verified.context.batch.cooling != 0.995f ||
|
||||
verified.context.batch.minimum_temperature != 0.005f ||
|
||||
verified.context.batch.jump_chance != 0.02f ||
|
||||
verified.context.batch.initial_state_jitter != 0.015f ||
|
||||
verified.context.batch.replica_temperature_ratio != 1.25f ||
|
||||
verified.context.batch.pbt_exploit_chance != 0.18f ||
|
||||
verified.context.batch.pbt_state_jitter != 0.012f ||
|
||||
verified.context.batch.injected_state_jitter != 0.009f ||
|
||||
verified.context.batch.strategy_weights[4] != 0.3f || verified.candidate_hash != 204 ||
|
||||
verified.candidate_state.values[35] != make_state(0.3f).values[35] ||
|
||||
verified.cuda_metrics.ambiguity_flags != 0x55 ||
|
||||
verified.rollout_rng_state != 205 ||
|
||||
verified.rollout_rng_spare_normal != -0.375f ||
|
||||
verified.rollout_rng_has_spare_normal != 1 ||
|
||||
verified.rollout_start_iterations != 20 ||
|
||||
verified.origin_batch_ordinal != 12 || verified.rollout_start_batch_ordinal != 13 ||
|
||||
verified.parent_batch_ordinal != 11 || verified.best_found_iteration != 180 ||
|
||||
verified.best_found_batch_ordinal != 17 || verified.origin_kind != 3 ||
|
||||
verified.origin_seed_index != 2 || verified.origin_pool_size != 8 ||
|
||||
verified.origin_seed_replica_count != 2 || verified.parent_chain_index != 4 ||
|
||||
verified.producing_chain_index != 6 ||
|
||||
fp32.context.record_sequence != 211 || fp32.state.values[35] != make_state(0.42f).values[35] ||
|
||||
fp32.approximate_metrics.energy != make_approximate().energy ||
|
||||
fp32.candidate_hash != 212 || fp32.chain_initial_rng_state != 213 ||
|
||||
fp32.rollout_rng_state != 214 || fp32.rollout_rng_spare_normal != 0.625f ||
|
||||
fp32.rollout_rng_has_spare_normal != 1 || fp32.chain_iterations != 215 ||
|
||||
fp32.rollout_start_iterations != 31 || fp32.origin_batch_ordinal != 21 ||
|
||||
fp32.rollout_start_batch_ordinal != 22 || fp32.parent_batch_ordinal != 20 ||
|
||||
fp32.best_found_iteration != 199 || fp32.best_found_batch_ordinal != 25 ||
|
||||
fp32.strategy != 3 || fp32.chain_index != 7 || fp32.producing_chain_index != 6 ||
|
||||
fp32.injected_seed_index != 1 || fp32.pool_size != 8 || fp32.replica_count != 2 ||
|
||||
fp32.origin_kind != 4 || fp32.origin_seed_index != 3 ||
|
||||
fp32.parent_chain_index != 5 || fp32.sampling_numerator != 1 ||
|
||||
fp32.sampling_denominator != 8 || fp32.sampling_probability != 0.125f ||
|
||||
fp32.flags != 0x41 ||
|
||||
proposal.purpose != SeedProposalPurpose::BatchInjection ||
|
||||
proposal.anchor_state.values[35] != make_state(0.51f).values[35] ||
|
||||
proposal.anchor_metrics.crossings != 7 ||
|
||||
proposal.seed_state.values[35] != make_state(0.53f).values[35] ||
|
||||
proposal.seed_metrics.intersections != 5 || proposal.neural_input[44] == 0.0f ||
|
||||
proposal.behavior.flags != 3 || proposal.plane_action != 4 ||
|
||||
proposal.move_action != 2 || proposal.scale_action != 1 ||
|
||||
proposal.plane_sampling_distribution[4] != 0.45f ||
|
||||
proposal.move_sampling_distribution[2] != 0.4f ||
|
||||
proposal.scale_sampling_distribution[1] != 0.7f ||
|
||||
proposal.plane_propensity != 0.45f || proposal.move_propensity != 0.4f ||
|
||||
proposal.scale_propensity != 0.7f || proposal.proposal_seed != 222 ||
|
||||
proposal.anchor_hash != 223 || proposal.seed_hash != 224 ||
|
||||
proposal.seed_index != 3 || proposal.pool_size != 8 ||
|
||||
proposal.assigned_chains != 2 ||
|
||||
proposal.flags != (SeedProposalHasNeuralInput | SeedProposalPredictionSupplied |
|
||||
SeedProposalNeuralGuided) ||
|
||||
injected.plane_action != 2 || injected.plane_propensity != 0.45f ||
|
||||
injected.plane_sampling_distribution[2] != 0.45f ||
|
||||
injected.rollout_seed != 305 || injected.rollout_rng_state != 307 ||
|
||||
injected.rollout_rng_spare_normal != -0.875f ||
|
||||
injected.rollout_rng_has_spare_normal != 1 ||
|
||||
injected.start_iterations != 33 || injected.origin_batch_ordinal != 19 ||
|
||||
injected.pool_size != 8 || injected.replica_count != 2 ||
|
||||
injected.selection_rule != 7 || injected.neural_input[44] == 0.0f ||
|
||||
(injected.flags & TrajectoryHasNeuralInput) == 0 ||
|
||||
rich.refinement_trajectories.front().steps.size() != 2 ||
|
||||
rich.refinement_trajectories.front().steps[0].iteration != 10 ||
|
||||
rich.refinement_trajectories.front().steps[0].perturbation == 0.0 ||
|
||||
rich.refinement_trajectories.front().steps[0].learning_rate == 0.0 ||
|
||||
rich.refinement_trajectories.front().steps[0].direction[0] != 1 ||
|
||||
rich.refinement_trajectories.front().steps[0].center_state.values[35] == 0.0 ||
|
||||
rich.refinement_trajectories.front().steps[0].plus_state.values[35] == 0.0 ||
|
||||
rich.refinement_trajectories.front().steps[0].minus_state.values[35] == 0.0 ||
|
||||
rich.refinement_trajectories.front().steps[0].updated_state.values[35] == 0.0 ||
|
||||
rich.refinement_trajectories.front().steps[0].plus_metrics.crossings != 3 ||
|
||||
rich.refinement_trajectories.front().steps[0].minus_metrics.intersections != 3 ||
|
||||
rich.refinement_trajectories.front().steps[0].flags !=
|
||||
(RefinementStepPlusEvaluated | RefinementStepMinusEvaluated |
|
||||
RefinementStepUpdateApplied | RefinementStepImproved) ||
|
||||
rich.refinement_trajectories.front().flags !=
|
||||
(RefinementAccepted | RefinementImprovedBest | RefinementEnteredArchive)) return fail(6);
|
||||
const std::array<RecordType, 7> expected_order{{
|
||||
RecordType::VerifiedCandidate,
|
||||
RecordType::Fp32Candidate,
|
||||
RecordType::SeedProposal,
|
||||
RecordType::InjectedTrajectory,
|
||||
RecordType::LegacyReplay,
|
||||
RecordType::RefinementTrajectory,
|
||||
RecordType::LegacyReplay,
|
||||
}};
|
||||
if (rich.record_order.size() != expected_order.size()) return fail(42);
|
||||
for (std::size_t index = 0; index < expected_order.size(); ++index) {
|
||||
if (rich.record_order[index].type != expected_order[index]) return fail(43);
|
||||
}
|
||||
if (rich.record_order[4].index_within_type != 0 ||
|
||||
rich.record_order[6].index_within_type != 1) return fail(44);
|
||||
|
||||
// Decoding and re-encoding every rich record must be byte-identical. This
|
||||
// catches fields that a partial hand-written round-trip assertion could
|
||||
// accidentally omit.
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(root / "rich_reencoded"));
|
||||
if (!writer.append(rich.verified_candidates[0]).accepted ||
|
||||
!writer.append(rich.fp32_candidates[0]).accepted ||
|
||||
!writer.append(rich.seed_proposals[0]).accepted ||
|
||||
!writer.append(rich.injected_trajectories[0]).accepted ||
|
||||
!writer.append(rich.legacy_replay[0]).accepted ||
|
||||
!writer.append(rich.refinement_trajectories[0]).accepted ||
|
||||
!writer.append(rich.legacy_replay[1]).accepted ||
|
||||
!writer.seal(&error)) {
|
||||
return fail(27, error);
|
||||
}
|
||||
}
|
||||
if (read_bytes(root / "rich" / "shard_00000000.sztd") !=
|
||||
read_bytes(root / "rich_reencoded" / "shard_00000000.sztd")) {
|
||||
return fail(28);
|
||||
}
|
||||
|
||||
// All policy actions must be present together and propensity must match the
|
||||
// actual normalized distribution used for sampling.
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(root / "invalid_policy"));
|
||||
InjectedTrajectory partial = make_injected();
|
||||
partial.move_action = -1;
|
||||
if (writer.append(partial).status != AppendStatus::InvalidRecord) return fail(7);
|
||||
InjectedTrajectory mismatch = make_injected();
|
||||
mismatch.plane_propensity = 0.2f;
|
||||
if (writer.append(mismatch).status != AppendStatus::InvalidRecord) return fail(8);
|
||||
VerifiedCandidate bad_ratio = make_verified();
|
||||
bad_ratio.context.batch.fresh_numerator = 2;
|
||||
bad_ratio.context.batch.fresh_denominator = 1;
|
||||
if (writer.append(bad_ratio).status != AppendStatus::InvalidRecord) return fail(45);
|
||||
VerifiedCandidate bad_evaluation_breakdown = make_verified();
|
||||
bad_evaluation_breakdown.context.batch.initialization_evaluated_states =
|
||||
bad_evaluation_breakdown.context.batch.evaluated_states + 1;
|
||||
if (writer.append(bad_evaluation_breakdown).status != AppendStatus::InvalidRecord) return fail(56);
|
||||
Fp32Candidate bad_probability = make_fp32_candidate();
|
||||
bad_probability.sampling_probability = 1.01f;
|
||||
if (writer.append(bad_probability).status != AppendStatus::InvalidRecord) return fail(46);
|
||||
Fp32Candidate bad_sampling_ratio = make_fp32_candidate();
|
||||
bad_sampling_ratio.sampling_denominator = 7;
|
||||
if (writer.append(bad_sampling_ratio).status != AppendStatus::InvalidRecord) return fail(49);
|
||||
VerifiedCandidate bad_rng_snapshot = make_verified();
|
||||
bad_rng_snapshot.rollout_rng_has_spare_normal = 0;
|
||||
if (writer.append(bad_rng_snapshot).status != AppendStatus::InvalidRecord) return fail(50);
|
||||
VerifiedCandidate failed_verification = make_verified();
|
||||
failed_verification.flags = CandidateSelectedForCpuVerification |
|
||||
CandidateVerificationFailed;
|
||||
failed_verification.verified_metrics = {};
|
||||
if (!writer.append(failed_verification).accepted) return fail(51);
|
||||
failed_verification.flags |= CandidateImprovedBest;
|
||||
if (writer.append(failed_verification).status != AppendStatus::InvalidRecord) return fail(52);
|
||||
SeedProposal bad_proposal = make_seed_proposal();
|
||||
bad_proposal.scale_propensity = 0.2f;
|
||||
if (writer.append(bad_proposal).status != AppendStatus::InvalidRecord) return fail(47);
|
||||
RefinementTrajectory bad_refinement = make_refinement();
|
||||
bad_refinement.steps[0].direction[0] = 0;
|
||||
if (writer.append(bad_refinement).status != AppendStatus::InvalidRecord) return fail(48);
|
||||
RefinementTrajectory failed_refinement = make_refinement();
|
||||
failed_refinement.flags = RefinementVerificationFailed;
|
||||
failed_refinement.result_state = failed_refinement.start_state;
|
||||
failed_refinement.result_metrics = failed_refinement.start_metrics;
|
||||
failed_refinement.result_hash = failed_refinement.start_hash;
|
||||
if (!writer.append(failed_refinement).accepted) return fail(54);
|
||||
failed_refinement.flags |= RefinementAccepted;
|
||||
if (writer.append(failed_refinement).status != AppendStatus::InvalidRecord) return fail(55);
|
||||
}
|
||||
|
||||
// A corrupted complete frame inside the last committed prefix is a hard
|
||||
// error and must not be silently truncated.
|
||||
const std::filesystem::path committed_corrupt = root / "committed_corrupt";
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(committed_corrupt));
|
||||
if (!writer.append(make_legacy(701)).accepted || !writer.flush(&error)) return fail(9, error);
|
||||
}
|
||||
if (!corrupt_byte(committed_corrupt / "active.sztd.wal", 48)) return fail(10);
|
||||
RecoveryReport report;
|
||||
if (recover_archive_wals(committed_corrupt, kTestCap, report, &error)) return fail(11);
|
||||
std::filesystem::remove_all(committed_corrupt, cleanup_error);
|
||||
|
||||
// Arbitrary corruption in a non-final frame of an uncommitted suffix is
|
||||
// discarded to the previous durable commit before semantic scanning.
|
||||
const std::filesystem::path suffix_root = root / "suffix";
|
||||
std::vector<char> old_commit;
|
||||
std::uint64_t committed_length = 0;
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(suffix_root));
|
||||
if (!writer.append(make_legacy(801)).accepted || !writer.flush(&error)) return fail(12, error);
|
||||
committed_length = std::filesystem::file_size(suffix_root / "active.sztd.wal");
|
||||
old_commit = read_bytes(suffix_root / "active.sztd.wal.commit");
|
||||
if (!writer.append(make_legacy(802)).accepted ||
|
||||
!writer.append(make_legacy(803)).accepted) return fail(13);
|
||||
} // destructor commits the suffix; restore the old marker to emulate power loss.
|
||||
if (!write_bytes(suffix_root / "active.sztd.wal.commit", old_commit) ||
|
||||
!corrupt_byte(suffix_root / "active.sztd.wal", committed_length + 8)) return fail(14);
|
||||
report = {};
|
||||
if (!recover_archive_wals(suffix_root, kTestCap, report, &error)) return fail(15, error);
|
||||
TrainingShard suffix_shard;
|
||||
if (!read_shard(suffix_root / "shard_00000000.sztd", suffix_shard, &error) ||
|
||||
suffix_shard.info.record_count != 1 || report.torn_bytes_discarded == 0) return fail(16, error);
|
||||
|
||||
// Exact final size is accepted, immediately sealed, and raises one notice.
|
||||
std::uint64_t pending_one_record = 0;
|
||||
{
|
||||
const std::filesystem::path measure = root / "measure";
|
||||
TrainingArchiveWriter writer(config_for(measure));
|
||||
if (!writer.append(make_legacy(901)).accepted) return fail(17);
|
||||
pending_one_record = writer.pending_bytes();
|
||||
}
|
||||
std::filesystem::remove_all(root / "measure", cleanup_error);
|
||||
const std::uint64_t exact_cap = pending_one_record + 40; // format-v3 footer size.
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(root / "exact_cap", exact_cap));
|
||||
if (!writer.append(make_legacy(902)).accepted || !writer.limit_reached()) return fail(18);
|
||||
std::string notice;
|
||||
if (!writer.take_limit_notice(notice) || writer.take_limit_notice(notice) ||
|
||||
writer.committed_bytes() != exact_cap ||
|
||||
writer.append(make_legacy(903)).status != AppendStatus::LimitReached) return fail(19);
|
||||
}
|
||||
std::uint64_t exact_cap_bytes = 0;
|
||||
if (!scan_archive_bytes(root / "exact_cap", exact_cap_bytes, &error) ||
|
||||
exact_cap_bytes != exact_cap) {
|
||||
return fail(29, error);
|
||||
}
|
||||
|
||||
// A second flush with no new records is a no-op for the commit sidecar.
|
||||
// Only 48 bytes remain after the first commit: too little for another
|
||||
// 56-byte temporary marker, but enough to seal the 40-byte footer.
|
||||
const std::uint64_t noop_flush_cap = pending_one_record + 56 + 48;
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(root / "noop_flush", noop_flush_cap));
|
||||
if (!writer.append(make_legacy(951)).accepted || !writer.flush(&error) ||
|
||||
!writer.flush(&error) || !writer.seal(&error) ||
|
||||
writer.committed_bytes() > noop_flush_cap ||
|
||||
writer.append(make_legacy(952)).status != AppendStatus::LimitReached) {
|
||||
return fail(20, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic commit replacement near a small cap: both dirty flushes and final
|
||||
// seal succeed without exceeding the cap.
|
||||
const std::uint64_t replacement_cap = pending_one_record * 3 + 4096;
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(root / "replace_commit", replacement_cap));
|
||||
if (!writer.append(make_legacy(1001)).accepted || !writer.flush(&error) ||
|
||||
!writer.append(make_legacy(1002)).accepted || !writer.flush(&error) ||
|
||||
!writer.seal(&error) || writer.committed_bytes() > replacement_cap) {
|
||||
return fail(21, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Commit artifacts without a WAL contain no records. Recovery removes both
|
||||
// the official marker and a crashed atomic-replacement temporary.
|
||||
const std::filesystem::path stale = root / "stale";
|
||||
std::filesystem::create_directories(stale);
|
||||
if (!write_bytes(stale / "active.sztd.wal.commit", {'s', 't', 'a', 'l', 'e'}) ||
|
||||
!write_bytes(stale / "active.sztd.wal.commit.tmp", {'t', 'm', 'p'})) return fail(22);
|
||||
report = {};
|
||||
if (!recover_archive_wals(stale, kTestCap, report, &error) ||
|
||||
std::filesystem::exists(stale / "active.sztd.wal.commit") ||
|
||||
std::filesystem::exists(stale / "active.sztd.wal.commit.tmp")) return fail(23, error);
|
||||
|
||||
// A conflicting destination is never replaced or modified.
|
||||
const std::filesystem::path collision = root / "collision";
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(collision));
|
||||
if (!writer.append(make_legacy(1101)).accepted || !writer.flush(&error)) return fail(24, error);
|
||||
const std::vector<char> sentinel{'n', 'o', 't', '-', 'a', '-', 's', 'h', 'a', 'r', 'd'};
|
||||
if (!write_bytes(collision / "shard_00000000.sztd", sentinel) || writer.seal(&error)) {
|
||||
return fail(25, error);
|
||||
}
|
||||
if (read_bytes(collision / "shard_00000000.sztd") != sentinel) return fail(26);
|
||||
}
|
||||
|
||||
// A crash while the very first uncommitted fixed-size header is being
|
||||
// written may leave 0..40 garbage bytes. No committed record can exist in
|
||||
// that state, so recovery removes it instead of permanently wedging start.
|
||||
const std::filesystem::path partial_headers = root / "partial_headers";
|
||||
for (std::size_t size : std::array<std::size_t, 4>{{0, 1, 39, 40}}) {
|
||||
const std::filesystem::path directory =
|
||||
partial_headers / ("bytes_" + std::to_string(size));
|
||||
std::filesystem::create_directories(directory);
|
||||
if (!write_bytes(
|
||||
directory / "active.sztd.wal",
|
||||
std::vector<char>(size, static_cast<char>(0x5a)))) {
|
||||
return fail(30);
|
||||
}
|
||||
}
|
||||
report = {};
|
||||
if (!recover_archive_wals(partial_headers, kTestCap, report, &error) ||
|
||||
report.empty_wals_removed != 4) {
|
||||
return fail(31, error);
|
||||
}
|
||||
std::uint64_t partial_bytes = 1;
|
||||
if (!scan_archive_bytes(partial_headers, partial_bytes, &error) || partial_bytes != 0) {
|
||||
return fail(32, error);
|
||||
}
|
||||
|
||||
// A valid commit marker makes even header corruption a hard error. The
|
||||
// recovery attempt must leave both artifacts byte-for-byte untouched.
|
||||
const std::filesystem::path committed_header = root / "committed_header";
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(committed_header));
|
||||
if (!writer.append(make_legacy(1201)).accepted || !writer.flush(&error)) {
|
||||
return fail(33, error);
|
||||
}
|
||||
}
|
||||
const std::filesystem::path committed_header_wal =
|
||||
committed_header / "active.sztd.wal";
|
||||
const std::filesystem::path committed_header_marker =
|
||||
committed_header / "active.sztd.wal.commit";
|
||||
if (!corrupt_byte(committed_header_wal, 0)) return fail(34);
|
||||
const std::vector<char> corrupted_header_bytes = read_bytes(committed_header_wal);
|
||||
const std::vector<char> committed_header_bytes = read_bytes(committed_header_marker);
|
||||
report = {};
|
||||
if (recover_archive_wals(committed_header, kTestCap, report, &error) ||
|
||||
read_bytes(committed_header_wal) != corrupted_header_bytes ||
|
||||
read_bytes(committed_header_marker) != committed_header_bytes) {
|
||||
return fail(35, error);
|
||||
}
|
||||
|
||||
// A fully committed WAL can exactly consume the remaining allocation
|
||||
// while still lacking its 40-byte immutable footer. This is a normal
|
||||
// cache-limit boundary: recovery must preserve the WAL and marker, and a
|
||||
// writer opened against the same accounting must freeze collection and
|
||||
// emit one limit notice rather than reporting an I/O failure.
|
||||
const std::filesystem::path deferred_at_limit = root / "deferred_at_limit";
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(deferred_at_limit));
|
||||
if (!writer.append(make_legacy(1251)).accepted || !writer.flush(&error)) {
|
||||
return fail(49, error);
|
||||
}
|
||||
}
|
||||
const std::filesystem::path deferred_wal =
|
||||
deferred_at_limit / "active.sztd.wal";
|
||||
const std::filesystem::path deferred_commit =
|
||||
deferred_at_limit / "active.sztd.wal.commit";
|
||||
const std::vector<char> deferred_wal_bytes = read_bytes(deferred_wal);
|
||||
const std::vector<char> deferred_commit_bytes = read_bytes(deferred_commit);
|
||||
std::uint64_t deferred_cap = 0;
|
||||
if (deferred_wal_bytes.empty() || deferred_commit_bytes.empty() ||
|
||||
!scan_archive_bytes(deferred_at_limit, deferred_cap, &error)) {
|
||||
return fail(50, error);
|
||||
}
|
||||
report = {};
|
||||
if (!recover_archive_wals(deferred_at_limit, deferred_cap, report, &error) ||
|
||||
report.wals_found != 1 || report.wals_deferred_at_limit != 1 ||
|
||||
report.shards_sealed != 0 || report.records_recovered != 0 ||
|
||||
read_bytes(deferred_wal) != deferred_wal_bytes ||
|
||||
read_bytes(deferred_commit) != deferred_commit_bytes ||
|
||||
std::filesystem::exists(
|
||||
deferred_at_limit / "shard_00000000.sztd")) {
|
||||
return fail(51, error);
|
||||
}
|
||||
WriterConfig deferred_config = config_for(deferred_at_limit, deferred_cap);
|
||||
deferred_config.initial_existing_bytes = deferred_cap;
|
||||
{
|
||||
TrainingArchiveWriter writer(deferred_config);
|
||||
std::string notice;
|
||||
if (!writer.limit_reached() || writer.collection_enabled() ||
|
||||
writer.has_write_error() || !writer.take_limit_notice(notice) ||
|
||||
notice.empty() || writer.take_limit_notice(notice) ||
|
||||
writer.append(make_legacy(1252)).status != AppendStatus::LimitReached ||
|
||||
!writer.flush(&error)) {
|
||||
return fail(52, error);
|
||||
}
|
||||
}
|
||||
if (read_bytes(deferred_wal) != deferred_wal_bytes ||
|
||||
read_bytes(deferred_commit) != deferred_commit_bytes) {
|
||||
return fail(53);
|
||||
}
|
||||
|
||||
// If an equivalent immutable shard already exists, the startup byte scan
|
||||
// counted both it and the leftover sealed WAL. Recount after deduplication
|
||||
// so a following committed WAL can use the room that was actually freed.
|
||||
const std::filesystem::path equivalent_root = root / "equivalent_recovery";
|
||||
const std::filesystem::path duplicate_dir = equivalent_root / "a_duplicate";
|
||||
const std::filesystem::path pending_dir = equivalent_root / "z_pending";
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(duplicate_dir));
|
||||
if (!writer.append(make_legacy(1301)).accepted || !writer.seal(&error)) {
|
||||
return fail(36, error);
|
||||
}
|
||||
}
|
||||
const std::filesystem::path duplicate_shard =
|
||||
duplicate_dir / "shard_00000000.sztd";
|
||||
const std::uint64_t duplicate_size = std::filesystem::file_size(duplicate_shard);
|
||||
if (!std::filesystem::copy_file(
|
||||
duplicate_shard,
|
||||
duplicate_dir / "active.sztd.wal",
|
||||
std::filesystem::copy_options::none,
|
||||
cleanup_error) || cleanup_error) {
|
||||
return fail(37, cleanup_error.message());
|
||||
}
|
||||
{
|
||||
TrainingArchiveWriter writer(config_for(pending_dir));
|
||||
if (!writer.append(make_legacy(1302)).accepted || !writer.flush(&error)) {
|
||||
return fail(38, error);
|
||||
}
|
||||
}
|
||||
std::uint64_t equivalent_initial_bytes = 0;
|
||||
if (!scan_archive_bytes(equivalent_root, equivalent_initial_bytes, &error) ||
|
||||
equivalent_initial_bytes < duplicate_size) {
|
||||
return fail(39, error);
|
||||
}
|
||||
const std::uint64_t recovery_cap =
|
||||
equivalent_initial_bytes - duplicate_size + 40; // one v3 footer
|
||||
report = {};
|
||||
if (!recover_archive_wals(equivalent_root, recovery_cap, report, &error) ||
|
||||
std::filesystem::exists(duplicate_dir / "active.sztd.wal") ||
|
||||
std::filesystem::exists(pending_dir / "active.sztd.wal") ||
|
||||
std::filesystem::exists(pending_dir / "active.sztd.wal.commit") ||
|
||||
!std::filesystem::exists(pending_dir / "shard_00000000.sztd")) {
|
||||
return fail(40, error);
|
||||
}
|
||||
std::uint64_t equivalent_final_bytes = 0;
|
||||
if (!scan_archive_bytes(equivalent_root, equivalent_final_bytes, &error) ||
|
||||
equivalent_final_bytes > recovery_cap) {
|
||||
return fail(41, error);
|
||||
}
|
||||
|
||||
std::filesystem::remove_all(root, cleanup_error);
|
||||
std::cout << "training archive self-test passed\n";
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
<Configuration>Debug</Configuration><Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration><Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|Win32">
|
||||
<Configuration>Release</Configuration><Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration><Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{17B67DC7-C0CE-4EDF-B2C6-6FB58B39DB74}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword><WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType><UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v145</PlatformToolset><CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType><UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v145</PlatformToolset><CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<PropertyGroup>
|
||||
<RepositoryRoot>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\..\'))</RepositoryRoot>
|
||||
<OutDir>$(RepositoryRoot)build\msbuild\bin\$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(RepositoryRoot)build\msbuild\obj\$(ProjectName)\$(Platform)\$(Configuration)\</IntDir>
|
||||
<LocalDebuggerWorkingDirectory>$(RepositoryRoot)</LocalDebuggerWorkingDirectory>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile><LanguageStandard>stdcpp17</LanguageStandard><WarningLevel>Level4</WarningLevel>
|
||||
<PreprocessorDefinitions>UNICODE;_UNICODE;NOMINMAX;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalOptions>/permissive- %(AdditionalOptions)</AdditionalOptions></ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="TrainingArchiveSelfTest.cpp" />
|
||||
<ClCompile Include="..\..\src\TrainingArchive\TrainingArchive.cpp" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
</Project>
|
||||
Reference in New Issue
Block a user