Experiments

tools/analyze_large_comparison.ps1

@@ -0,0 +1,159 @@
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$InputDirectory
+)
+
+$ErrorActionPreference = 'Stop'
+$invariant = [Globalization.CultureInfo]::InvariantCulture
+[Threading.Thread]::CurrentThread.CurrentCulture = $invariant
+[Threading.Thread]::CurrentThread.CurrentUICulture = $invariant
+
+function ConvertTo-Number($Value) {
+    if ($Value -is [double] -or $Value -is [float] -or $Value -is [decimal]) {
+        return [double]$Value
+    }
+    $text = ([string]$Value).Trim()
+    if ($text.Contains(',') -and -not $text.Contains('.')) {
+        $text = $text.Replace(',', '.')
+    }
+    return [double]::Parse($text, [Globalization.NumberStyles]::Float, $invariant)
+}
+
+$directory = if ([IO.Path]::IsPathRooted($InputDirectory)) {
+    [IO.Path]::GetFullPath($InputDirectory)
+} else {
+    [IO.Path]::GetFullPath((Join-Path (Resolve-Path '.').Path $InputDirectory))
+}
+$input = Join-Path $directory 'cell-medians-large.csv'
+if (-not (Test-Path -LiteralPath $input -PathType Leaf)) {
+    throw "Missing cell medians: $input"
+}
+
+function Get-GeometricMean($Values) {
+    $positive = @($Values | Where-Object { (ConvertTo-Number $_) -gt 0 })
+    if ($positive.Count -eq 0) { return [double]::NaN }
+    $meanLog = ($positive | ForEach-Object {
+        [math]::Log((ConvertTo-Number $_))
+    } | Measure-Object -Average).Average
+    return [math]::Exp($meanLog)
+}
+
+$medians = @(Import-Csv -LiteralPath $input)
+$comparisons = @($medians | Group-Object profile,type,workload,n | ForEach-Object {
+    $rows = @($_.Group)
+    $fixed = @($rows | Where-Object {
+        $_.structure -like 'tiered_forced_leaf_*'
+    } | Sort-Object { ConvertTo-Number $_.median_ns_per_op })
+    $adaptive = @($rows | Where-Object {
+        $_.structure -like 'adaptive_shape_deferred*'
+    })[0]
+    $vector = @($rows | Where-Object {
+        $_.structure -eq 'std::vector_reserved'
+    })[0]
+    if ($fixed.Count -ne 5 -or $null -eq $adaptive -or $null -eq $vector) {
+        throw "Incomplete seven-candidate cell: $($_.Name)"
+    }
+
+    $best = $fixed[0]
+    $ppmMatch = [regex]::Match($adaptive.workload, '_edit_ppm_(\d+)$')
+    if (-not $ppmMatch.Success) { throw "Invalid large workload: $($adaptive.workload)" }
+    $editPpm = [int]$ppmMatch.Groups[1].Value
+    $pattern = $adaptive.workload.Substring(
+        0, $adaptive.workload.Length - $ppmMatch.Value.Length)
+    $bestLeaf = [regex]::Match($best.structure, 'leaf_(\d+)').Groups[1].Value
+    $fixedNs = ConvertTo-Number $best.median_ns_per_op
+    $adaptiveNs = ConvertTo-Number $adaptive.median_ns_per_op
+    $vectorNs = ConvertTo-Number $vector.median_ns_per_op
+
+    [pscustomobject]@{
+        profile = $adaptive.profile
+        type = $adaptive.type
+        n = [long]$adaptive.n
+        pattern = $pattern
+        edit_ppm = $editPpm
+        workload = $adaptive.workload
+        best_fixed_leaf = [int]$bestLeaf
+        best_fixed_ns_per_op = $fixedNs
+        adaptive_ns_per_op = $adaptiveNs
+        vector_ns_per_op = $vectorNs
+        adaptive_over_best_fixed = $adaptiveNs / $fixedNs
+        vector_over_best_fixed = $vectorNs / $fixedNs
+        adaptive_over_vector = $adaptiveNs / $vectorNs
+        adaptive_switches = ConvertTo-Number $adaptive.median_switches
+        adaptive_leaf_rebuilds = ConvertTo-Number $adaptive.median_leaf_rebuilds
+        adaptive_final_mode = $adaptive.final_mode
+        adaptive_final_leaf = [int]$adaptive.final_leaf
+        adaptive_transition_ns = ConvertTo-Number $adaptive.median_max_transition_ns
+        adaptive_construction_ns = ConvertTo-Number $adaptive.median_construction_ns
+        vector_construction_ns = ConvertTo-Number $vector.median_construction_ns
+        best_fixed_construction_ns = ConvertTo-Number $best.median_construction_ns
+    }
+})
+
+$comparisons | Sort-Object n,pattern,edit_ppm,profile |
+    Export-Csv -LiteralPath (Join-Path $directory 'comparison-large.csv') `
+        -NoTypeInformation -Encoding utf8
+
+$bySize = @($comparisons | Group-Object n | ForEach-Object {
+    $cells = @($_.Group)
+    [pscustomobject]@{
+        n = [long]$cells[0].n
+        cells = $cells.Count
+        gm_adaptive_over_best_fixed = Get-GeometricMean $cells.adaptive_over_best_fixed
+        gm_vector_over_best_fixed = Get-GeometricMean $cells.vector_over_best_fixed
+        gm_adaptive_over_vector = Get-GeometricMean $cells.adaptive_over_vector
+        adaptive_beats_best_fixed = @($cells | Where-Object {
+            $_.adaptive_over_best_fixed -lt 1
+        }).Count
+        adaptive_beats_vector = @($cells | Where-Object {
+            $_.adaptive_over_vector -lt 1
+        }).Count
+        adaptive_final_vector = @($cells | Where-Object {
+            $_.adaptive_final_mode -eq 'vector'
+        }).Count
+        adaptive_final_tiered = @($cells | Where-Object {
+            $_.adaptive_final_mode -eq 'tiered'
+        }).Count
+    }
+})
+$bySize | Sort-Object n | Export-Csv `
+    -LiteralPath (Join-Path $directory 'comparison-by-size.csv') `
+    -NoTypeInformation -Encoding utf8
+
+$byWorkload = @($comparisons | Group-Object n,pattern,edit_ppm | ForEach-Object {
+    $cells = @($_.Group)
+    [pscustomobject]@{
+        n = [long]$cells[0].n
+        pattern = $cells[0].pattern
+        edit_ppm = [int]$cells[0].edit_ppm
+        profiles = $cells.Count
+        best_fixed_leaf_mode = ($cells | Group-Object best_fixed_leaf |
+            Sort-Object Count -Descending | Select-Object -First 1).Name
+        gm_adaptive_over_best_fixed = Get-GeometricMean $cells.adaptive_over_best_fixed
+        gm_vector_over_best_fixed = Get-GeometricMean $cells.vector_over_best_fixed
+        gm_adaptive_over_vector = Get-GeometricMean $cells.adaptive_over_vector
+        mean_switches = ($cells.adaptive_switches | Measure-Object -Average).Average
+        mean_leaf_rebuilds = ($cells.adaptive_leaf_rebuilds | Measure-Object -Average).Average
+        final_modes = (($cells.adaptive_final_mode | Group-Object | ForEach-Object {
+            "$($_.Name):$($_.Count)"
+        }) -join ';')
+    }
+})
+$byWorkload | Sort-Object n,pattern,edit_ppm | Export-Csv `
+    -LiteralPath (Join-Path $directory 'comparison-by-workload.csv') `
+    -NoTypeInformation -Encoding utf8
+
+$winners = @($comparisons | Group-Object best_fixed_leaf | ForEach-Object {
+    [pscustomobject]@{
+        leaf = [int]$_.Name
+        wins = $_.Count
+        percent = 100.0 * $_.Count / $comparisons.Count
+    }
+})
+$winners | Sort-Object wins -Descending | Export-Csv `
+    -LiteralPath (Join-Path $directory 'fixed-leaf-winners.csv') `
+    -NoTypeInformation -Encoding utf8
+
+Write-Host "Large comparison analysis: $directory"
+$bySize | Sort-Object n | Format-Table -AutoSize
+$winners | Sort-Object wins -Descending | Format-Table -AutoSize

tools/analyze_results.ps1

@@ -53,14 +53,14 @@ function Get-Mode($Values) {
 # Every implementation must produce the same observable result for an
 # identical trace.  A mismatch invalidates the performance run.
 $mismatches = @($rows |
-    Group-Object profile,type,workload,n,repeat,seed |
+    Group-Object profile,type,workload,n,operations,repeat,seed |
     Where-Object { @($_.Group.checksum | Select-Object -Unique).Count -ne 1 })
 if ($mismatches.Count -ne 0) {
     throw "Checksum mismatch in $($mismatches.Count) benchmark cell(s)"
 }
 
 $medians = @($rows |
-    Group-Object profile,type,structure,workload,n,leaf,fanout,levels |
+    Group-Object profile,type,structure,workload,n,operations,leaf,fanout,levels |
     ForEach-Object {
         $group = $_.Group
         $candidate = $group[0].structure
@@ -73,6 +73,7 @@ $medians = @($rows |
             structure = $candidate
             workload = $group[0].workload
             n = [uint64]$group[0].n
+            operations = [uint64]$group[0].operations
             leaf = [uint64]$group[0].leaf
             fanout = [uint64]$group[0].fanout
             levels = [uint64]$group[0].levels
@@ -80,6 +81,8 @@ $medians = @($rows |
             final_fanout = Get-Mode @($group.final_fanout)
             final_levels = Get-Mode @($group.final_levels)
             final_mode = Get-Mode @($group.final_mode)
+            median_construction_ns = Get-Median @($group.construction_ns | ForEach-Object { [double]$_ })
+            median_initial_bytes = Get-Median @($group.initial_bytes | ForEach-Object { [double]$_ })
             median_ns = Get-Median @($group.total_ns | ForEach-Object { [double]$_ })
             median_ns_per_op = Get-Median @($group.ns_per_op | ForEach-Object { [double]$_ })
             median_bytes = Get-Median @($group.allocated_bytes | ForEach-Object { [double]$_ })
@@ -103,24 +106,24 @@ $medians = @($rows |
         }
     })
 
-$medians | Sort-Object profile,type,workload,n,structure |
+$medians | Sort-Object profile,type,workload,n,operations,structure |
     Export-Csv -LiteralPath $CellMedianFile -NoTypeInformation -Encoding utf8
 
 $fixedNames = @('std::vector_reserved', 'std::deque',
                 'std::list_index_semantics')
 $bestFixed = @{}
-$medians | Group-Object profile,type,workload,n | ForEach-Object {
+$medians | Group-Object profile,type,workload,n,operations | ForEach-Object {
     $eligible = @($_.Group | Where-Object {
         $fixedNames -contains $_.structure -or $_.structure -like 'tiered_forced_leaf_*'
     })
     if ($eligible.Count -gt 0) {
-        $key = "$($_.Group[0].profile)|$($_.Group[0].type)|$($_.Group[0].workload)|$($_.Group[0].n)"
+        $key = "$($_.Group[0].profile)|$($_.Group[0].type)|$($_.Group[0].workload)|$($_.Group[0].n)|$($_.Group[0].operations)"
         $bestFixed[$key] = ($eligible.median_ns | Measure-Object -Minimum).Minimum
     }
 }
 
 $cellScores = @($medians | ForEach-Object {
-    $key = "$($_.profile)|$($_.type)|$($_.workload)|$($_.n)"
+    $key = "$($_.profile)|$($_.type)|$($_.workload)|$($_.n)|$($_.operations)"
     if ($bestFixed.ContainsKey($key) -and $_.median_ns -gt 0) {
         [pscustomobject]@{
             profile = $_.profile
@@ -128,6 +131,7 @@ $cellScores = @($medians | ForEach-Object {
             type = $_.type
             workload = $_.workload
             n = $_.n
+            operations = $_.operations
             score = [double]$bestFixed[$key] / [double]$_.median_ns
         }
     }

tools/run_large_parallel.ps1

@@ -0,0 +1,414 @@
+param(
+    [ValidateSet('smoke', 'full')]
+    [string]$Scale = 'full',
+
+    [ValidateRange(2, 12)]
+    [int]$Workers = 12,
+
+    [ValidateRange(1, 12)]
+    [int]$MaxHugeWorkers = 3,
+
+    [switch]$OnlyHuge,
+
+    [string]$ResultDirectory = '',
+
+    [string]$Profiles = 'scalar baseline avx2'
+)
+
+$ErrorActionPreference = 'Stop'
+$projectRoot = (Resolve-Path '.').Path
+if (-not $ResultDirectory) {
+    $runId = "$(Get-Date -Format yyyyMMdd-HHmmss)-large-$Scale"
+    $ResultDirectory = Join-Path 'results\benchmarks' $runId
+}
+$result = if ([IO.Path]::IsPathRooted($ResultDirectory)) {
+    [IO.Path]::GetFullPath($ResultDirectory)
+} else {
+    [IO.Path]::GetFullPath((Join-Path $projectRoot $ResultDirectory))
+}
+$runId = Split-Path -Leaf $result
+$partials = Join-Path $result 'partials'
+$logs = Join-Path $result 'logs'
+New-Item -ItemType Directory -Force -Path $partials, $logs | Out-Null
+
+$profileList = @($Profiles -split '[, ]+' | Where-Object { $_ })
+if ($profileList.Count -eq 0) { throw 'At least one profile is required.' }
+foreach ($profile in $profileList) {
+    $binary = Join-Path $projectRoot "out\bin\$profile\Release\uc_bench.exe"
+    if (-not (Test-Path -LiteralPath $binary -PathType Leaf)) {
+        throw "Missing benchmark binary for profile '$profile': $binary"
+    }
+}
+
+function New-Case([string]$Pattern, [int]$EditPpm) {
+    [pscustomobject]@{ pattern = $Pattern; edit_ppm = $EditPpm }
+}
+
+if ($Scale -eq 'smoke') {
+    $sizes = @(100000, 1000000)
+    $cases = @(
+        New-Case 'steady_uniform' 0
+        New-Case 'steady_uniform' 10000
+        New-Case 'steady_uniform' 200000
+        New-Case 'steady_uniform' 1000000
+        New-Case 'steady_localized' 200000
+        New-Case 'bursty_uniform' 10000
+        New-Case 'phase' 200000
+    )
+} else {
+    $sizes = @(100000, 1000000, 10000000, 100000000)
+    $cases = @(
+        New-Case 'steady_uniform' 0
+        New-Case 'steady_uniform' 100
+        New-Case 'steady_uniform' 1000
+        New-Case 'steady_uniform' 10000
+        New-Case 'steady_uniform' 50000
+        New-Case 'steady_uniform' 200000
+        New-Case 'steady_uniform' 500000
+        New-Case 'steady_uniform' 1000000
+        New-Case 'steady_localized' 1000
+        New-Case 'steady_localized' 10000
+        New-Case 'steady_localized' 200000
+        New-Case 'steady_localized' 1000000
+        New-Case 'bursty_uniform' 10000
+        New-Case 'phase' 200000
+    )
+}
+
+if ($OnlyHuge) {
+    if ($Scale -ne 'full') { throw '-OnlyHuge requires -Scale full.' }
+    $sizes = @(100000000)
+}
+
+function Get-TargetEdits([long]$N) {
+    if ($N -le 1000000) { return 1024L }
+    return 512L
+}
+
+function Get-Operations([long]$N, [int]$EditPpm) {
+    if ($EditPpm -eq 0) {
+        if ($N -le 1000000) { return 2000000L }
+        if ($N -le 10000000) { return 1000000L }
+        return 500000L
+    }
+    $operations = [long][math]::Ceiling(
+        (Get-TargetEdits $N) * 1000000.0 / $EditPpm)
+    return [math]::Min(8000000L, [math]::Max(1L, $operations))
+}
+
+function Get-Repeats([long]$N) {
+    if ($N -le 1000000) { return 2 }
+    return 1
+}
+
+$jobs = @()
+foreach ($n in $sizes) {
+    foreach ($case in $cases) {
+        $operations = Get-Operations $n $case.edit_ppm
+        $repeats = Get-Repeats $n
+        foreach ($profile in $profileList) {
+            $name = '{0}-n{1}-{2}-ppm{3}' -f `
+                $profile, $n, $case.pattern, $case.edit_ppm
+            $jobs += [pscustomobject]@{
+                profile = $profile
+                n = [long]$n
+                pattern = $case.pattern
+                edit_ppm = [int]$case.edit_ppm
+                operations = [long]$operations
+                repeats = [int]$repeats
+                name = $name
+                output = Join-Path $partials "$name.csv"
+                summary = Join-Path $partials "$name-summary.csv"
+                stdout = Join-Path $logs "$name.out.log"
+                stderr = Join-Path $logs "$name.err.log"
+            }
+        }
+    }
+}
+
+# Keep profiles of one cell adjacent, but interleave N=100m with smaller sizes.
+# Twelve simultaneous huge adaptive rebuilds could consume almost all 32 GiB;
+# this ordering still occupies all cores without deliberately forcing paging.
+$jobs = @($jobs | Sort-Object pattern, edit_ppm,
+    @{ Expression = 'n'; Descending = $true }, profile)
+
+function Test-Partial($Job) {
+    if (-not (Test-Path -LiteralPath $Job.output -PathType Leaf)) { return $false }
+    try {
+        $rows = @(Import-Csv -LiteralPath $Job.output)
+        if ($rows.Count -ne 7 * $Job.repeats) { return $false }
+        $expectedWorkload = "$($Job.pattern)_edit_ppm_$($Job.edit_ppm)"
+        $invalidRows = @($rows | Where-Object {
+            $_.status -ne 'ok' -or $_.profile -ne $Job.profile -or
+            $_.suite -ne 'large' -or $_.type -ne 'uint32' -or
+            [long]$_.n -ne $Job.n -or
+            [long]$_.operations -ne $Job.operations -or
+            $_.workload -ne $expectedWorkload
+        })
+        if ($invalidRows.Count -ne 0) {
+            return $false
+        }
+        $expectedCandidates = @(
+            'adaptive_shape_deferred', 'std::vector_reserved',
+            'tiered_forced_leaf_64', 'tiered_forced_leaf_128',
+            'tiered_forced_leaf_256', 'tiered_forced_leaf_512',
+            'tiered_forced_leaf_1024'
+        ) | Sort-Object
+        for ($repeat = 0; $repeat -lt $Job.repeats; ++$repeat) {
+            $repeatRows = @($rows | Where-Object { [int]$_.repeat -eq $repeat })
+            if ($repeatRows.Count -ne 7) { return $false }
+            $actualCandidates = @($repeatRows.structure | Sort-Object -Unique)
+            if (($actualCandidates -join '|') -ne
+                ($expectedCandidates -join '|')) { return $false }
+            if (@($repeatRows.seed | Sort-Object -Unique).Count -ne 1 -or
+                @($repeatRows.checksum | Sort-Object -Unique).Count -ne 1) {
+                return $false
+            }
+        }
+        return $true
+    } catch {
+        return $false
+    }
+}
+
+function Get-TextSha256([string]$Text) {
+    $algorithm = [Security.Cryptography.SHA256]::Create()
+    try {
+        return -join ($algorithm.ComputeHash([Text.Encoding]::UTF8.GetBytes($Text)) |
+            ForEach-Object { $_.ToString('x2') })
+    } finally {
+        $algorithm.Dispose()
+    }
+}
+
+$binaryContract = @($profileList | ForEach-Object {
+    $binaryPath = Join-Path $projectRoot "out\bin\$_\Release\uc_bench.exe"
+    [ordered]@{
+        profile = $_
+        sha256 = (Get-FileHash -LiteralPath $binaryPath -Algorithm SHA256).Hash.ToLowerInvariant()
+    }
+})
+$contract = [ordered]@{
+    version = 1
+    scale = $Scale
+    workers = $Workers
+    max_huge_workers = $MaxHugeWorkers
+    only_huge = [bool]$OnlyHuge
+    profiles = $profileList
+    binaries = $binaryContract
+    jobs = @($jobs | Select-Object profile,n,pattern,edit_ppm,operations,repeats,name)
+}
+$planSignature = Get-TextSha256 ($contract | ConvertTo-Json -Depth 8 -Compress)
+$planPath = Join-Path $result 'large-plan.json'
+$isResume = Test-Path -LiteralPath $planPath -PathType Leaf
+if ($isResume) {
+    $existingPlan = Get-Content -Raw -LiteralPath $planPath | ConvertFrom-Json
+    if (-not $existingPlan.plan_signature -or
+        $existingPlan.plan_signature -ne $planSignature) {
+        throw 'ResultDirectory belongs to a different plan or benchmark binary; choose a new directory.'
+    }
+}
+
+if (-not $isResume) {
+    & powershell.exe -NoProfile -ExecutionPolicy Bypass `
+        -File 'tools\write_benchmark_environment.ps1' `
+        -ResultDirectory $result -RunId $runId -Scale "large-$Scale-parallel" `
+        -Profiles ($profileList -join ' ')
+    if ($LASTEXITCODE -ne 0) { throw 'Could not write benchmark environment.' }
+}
+
+$plan = [ordered]@{
+    run_id = $runId
+    generated_at = (Get-Date).ToString('o')
+    plan_signature = $planSignature
+    workers = $Workers
+    max_huge_workers = $MaxHugeWorkers
+    only_huge = [bool]$OnlyHuge
+    logical_processors = [Environment]::ProcessorCount
+    affinity_policy = 'one distinct logical CPU per worker slot'
+    profiles = $profileList
+    binaries = $binaryContract
+    candidates = @('std::vector_reserved', 'tiered_forced_leaf_64',
+        'tiered_forced_leaf_128', 'tiered_forced_leaf_256',
+        'tiered_forced_leaf_512', 'tiered_forced_leaf_1024',
+        'adaptive_shape_deferred')
+    sizes = $sizes
+    cases = $cases
+    operation_cap = 8000000
+    target_edits = [ordered]@{ n_le_1m = 1024; n_10m = 512; n_100m = 512 }
+    notes = @(
+        'Cells run concurrently; candidates inside one cell run sequentially on one trace.',
+        'Construction time is recorded separately and excluded from ns_per_op.',
+        'Parallel memory-bandwidth contention is accepted for exploratory throughput.',
+        'At most MaxHugeWorkers N=100m cells run together to avoid paging.',
+        'Target edits are upper goals; the 8m operation cap can reduce very sparse cells.',
+        'All seven candidates are mandatory; std::vector is never cost-guard skipped.'
+    )
+    jobs = @($jobs | Select-Object profile,n,pattern,edit_ppm,operations,repeats,name)
+}
+if (-not $isResume) {
+    $plan | ConvertTo-Json -Depth 8 |
+        Set-Content -LiteralPath $planPath -Encoding utf8
+}
+
+$completed = 0
+$pending = [Collections.ArrayList]::new()
+foreach ($job in $jobs) {
+    if (Test-Partial $job) {
+        ++$completed
+    } else {
+        [void]$pending.Add($job)
+    }
+}
+$total = $jobs.Count
+$running = @()
+$logical = [Environment]::ProcessorCount
+$cpuSlots = @(for ($slot = 0; $slot -lt $Workers; ++$slot) {
+    [math]::Min($logical - 1,
+        [int][math]::Floor(($slot + 0.5) * $logical / $Workers))
+})
+
+Write-Host ("Progress {0:N1}% ({1}/{2}); workers={3}; result={4}" -f `
+    (100.0 * $completed / $total), $completed, $total, $Workers, $result)
+
+try {
+    while ($pending.Count -ne 0 -or $running.Count -ne 0) {
+        while ($pending.Count -ne 0 -and $running.Count -lt $Workers) {
+            $hugeRunning = @($running | Where-Object {
+                $_.job.n -ge 100000000
+            }).Count
+            $nextIndex = -1
+            for ($index = 0; $index -lt $pending.Count; ++$index) {
+                if ($pending[$index].n -lt 100000000 -or
+                    $hugeRunning -lt $MaxHugeWorkers) {
+                    $nextIndex = $index
+                    break
+                }
+            }
+            if ($nextIndex -lt 0) { break }
+            $job = $pending[$nextIndex]
+            $pending.RemoveAt($nextIndex)
+            $binary = Join-Path $projectRoot `
+                "out\bin\$($job.profile)\Release\uc_bench.exe"
+            $arguments = @(
+                '--suite', 'large', '--scale', 'smoke',
+                '--n', [string]$job.n,
+                '--operations', [string]$job.operations,
+                '--edit-ppm', [string]$job.edit_ppm,
+                '--pattern', $job.pattern,
+                '--leaf', '512', '--fanout', '64', '--levels', '4',
+                '--repeats', [string]$job.repeats,
+                '--output', $job.output,
+                '--summary', $job.summary
+            )
+            # Start-Process enumerates the inherited environment and fails when
+            # VS has supplied both Path and PATH.  ProcessStartInfo inherits the
+            # environment directly and avoids that Windows PowerShell 5.1 bug.
+            $quotedArguments = @($arguments | ForEach-Object {
+                $argumentText = [string]$_
+                if ($argumentText -match '[\s"]') {
+                    '"' + ($argumentText -replace '"', '\"') + '"'
+                } else {
+                    $argumentText
+                }
+            }) -join ' '
+            $startInfo = [Diagnostics.ProcessStartInfo]::new()
+            $startInfo.FileName = $binary
+            $startInfo.Arguments = $quotedArguments
+            $startInfo.UseShellExecute = $false
+            $startInfo.CreateNoWindow = $true
+            $startInfo.RedirectStandardOutput = $true
+            $startInfo.RedirectStandardError = $true
+            $process = [Diagnostics.Process]::new()
+            $process.StartInfo = $startInfo
+            if (-not $process.Start()) {
+                throw "Could not start large job: $($job.name)"
+            }
+            $usedSlots = @($running | ForEach-Object { $_.slot })
+            $slot = 0
+            while ($usedSlots -contains $slot) { ++$slot }
+            try {
+                $cpu = $cpuSlots[$slot]
+                $process.ProcessorAffinity = [IntPtr](1L -shl $cpu)
+            } catch {
+                Add-Content -LiteralPath $job.stderr `
+                    -Value "Affinity warning: $($_.Exception.Message)"
+            }
+            $running += [pscustomobject]@{
+                process = $process
+                job = $job
+                slot = $slot
+            }
+        }
+
+        Start-Sleep -Milliseconds 250
+        $stillRunning = @()
+        foreach ($entry in $running) {
+            if (-not $entry.process.HasExited) {
+                $stillRunning += $entry
+                continue
+            }
+            $entry.process.WaitForExit()
+            $exitCode = $entry.process.ExitCode
+            $stdoutText = $entry.process.StandardOutput.ReadToEnd()
+            $stderrText = $entry.process.StandardError.ReadToEnd()
+            Set-Content -LiteralPath $entry.job.stdout -Value $stdoutText
+            Set-Content -LiteralPath $entry.job.stderr -Value $stderrText
+            $partialValid = Test-Partial $entry.job
+            # Windows PowerShell may expose a null ExitCode for a very short
+            # redirected process even after WaitForExit.  A fully validated
+            # 7*candidate CSV is authoritative in that narrow case.
+            if (($null -ne $exitCode -and $exitCode -ne 0) -or -not $partialValid) {
+                throw "Large job failed: $($entry.job.name); exit=$exitCode; $stderrText"
+            }
+            ++$completed
+            Write-Host ("Progress {0:N1}% ({1}/{2}) completed {3}" -f `
+                (100.0 * $completed / $total), $completed, $total, $entry.job.name)
+        }
+        $running = $stillRunning
+    }
+} catch {
+    foreach ($entry in $running) {
+        if (-not $entry.process.HasExited) { $entry.process.Kill() }
+    }
+    throw
+}
+
+foreach ($job in $jobs) {
+    if (-not (Test-Partial $job)) {
+        throw "Partial became invalid before merge: $($job.name)"
+    }
+}
+$rawRows = @($jobs | ForEach-Object {
+    Import-Csv -LiteralPath $_.output
+})
+$raw = Join-Path $result 'large-raw.csv'
+$rawRows | Export-Csv -LiteralPath $raw -NoTypeInformation -Encoding utf8
+
+& powershell.exe -NoProfile -ExecutionPolicy Bypass `
+    -File 'tools\analyze_results.ps1' -InputDirectory $result `
+    -Pattern 'large-raw.csv' `
+    -OutputFile (Join-Path $result 'aggregate-large.csv') `
+    -CellMedianFile (Join-Path $result 'cell-medians-large.csv')
+if ($LASTEXITCODE -ne 0) { throw 'Large result analysis failed.' }
+
+& powershell.exe -NoProfile -ExecutionPolicy Bypass `
+    -File 'tools\analyze_large_comparison.ps1' -InputDirectory $result
+if ($LASTEXITCODE -ne 0) { throw 'Large comparison analysis failed.' }
+
+@(
+    'powershell.exe -NoProfile -ExecutionPolicy Bypass -File tools\analyze_results.ps1 -InputDirectory <run> -Pattern large-raw.csv -OutputFile <run>\aggregate-large.csv -CellMedianFile <run>\cell-medians-large.csv',
+    'powershell.exe -NoProfile -ExecutionPolicy Bypass -File tools\analyze_large_comparison.ps1 -InputDirectory <run>'
+) | Set-Content -LiteralPath (Join-Path $result 'analysis-command.txt') -Encoding ascii
+
+$manifestPath = Join-Path $result 'results-manifest.sha256'
+$manifestLines = @(Get-ChildItem -LiteralPath $result -Recurse -File |
+    Where-Object { $_.FullName -ne $manifestPath } |
+    Sort-Object FullName | ForEach-Object {
+        $relative = $_.FullName.Substring($result.Length).TrimStart('\') -replace '\\', '/'
+        $hash = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
+        "$hash  $relative"
+    })
+$manifestLines | Set-Content -LiteralPath $manifestPath -Encoding ascii
+
+Write-Host "Progress 100.0% ($total/$total). Large benchmark completed: $result"

tools/write_benchmark_environment.ps1

@@ -6,7 +6,11 @@ param(
 )
 
 $ErrorActionPreference = 'Stop'
-$result = [IO.Path]::GetFullPath((Join-Path (Get-Location) $ResultDirectory))
+$result = if ([IO.Path]::IsPathRooted($ResultDirectory)) {
+    [IO.Path]::GetFullPath($ResultDirectory)
+} else {
+    [IO.Path]::GetFullPath((Join-Path (Get-Location) $ResultDirectory))
+}
 New-Item -ItemType Directory -Force -Path $result | Out-Null
 $cpu = (Get-ItemProperty 'HKLM:\HARDWARE\DESCRIPTION\System\CentralProcessor\0').ProcessorNameString
 $windows = Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion'
@@ -60,7 +64,8 @@ $metadata.GetEnumerator() | ForEach-Object {
 # the source set; generated executables receive their own manifest.
 $sourceRoots = @(
     '.gitignore', 'CMakeLists.txt', 'CMakePresets.json', 'README.md',
-    'build.bat', 'benchmark.bat', 'include', 'src', 'tests', 'tools', 'docs'
+    'build.bat', 'benchmark.bat', 'large_benchmark.bat',
+    'include', 'src', 'tests', 'tools', 'docs'
 )
 $sourceFiles = @()
 foreach ($entry in $sourceRoots) {