UniversalContainer/tools/run_large_parallel.ps1

param(
    [ValidateSet('smoke', 'full')]
    [string]$Scale = 'full',

    [ValidateRange(2, 12)]
    [int]$Workers = 12,

    [ValidateRange(1, 12)]
    [int]$MaxHugeWorkers = 3,

    [switch]$OnlyHuge,

    [string]$ResultDirectory = '',

    [string]$Profiles = 'scalar baseline avx2'
)

$ErrorActionPreference = 'Stop'
$projectRoot = (Resolve-Path '.').Path
if (-not $ResultDirectory) {
    $runId = "$(Get-Date -Format yyyyMMdd-HHmmss)-large-$Scale"
    $ResultDirectory = Join-Path 'results\benchmarks' $runId
}
$result = if ([IO.Path]::IsPathRooted($ResultDirectory)) {
    [IO.Path]::GetFullPath($ResultDirectory)
} else {
    [IO.Path]::GetFullPath((Join-Path $projectRoot $ResultDirectory))
}
$runId = Split-Path -Leaf $result
$partials = Join-Path $result 'partials'
$logs = Join-Path $result 'logs'
New-Item -ItemType Directory -Force -Path $partials, $logs | Out-Null

$profileList = @($Profiles -split '[, ]+' | Where-Object { $_ })
if ($profileList.Count -eq 0) { throw 'At least one profile is required.' }
foreach ($profile in $profileList) {
    $binary = Join-Path $projectRoot "out\bin\$profile\Release\uc_bench.exe"
    if (-not (Test-Path -LiteralPath $binary -PathType Leaf)) {
        throw "Missing benchmark binary for profile '$profile': $binary"
    }
}

function New-Case([string]$Pattern, [int]$EditPpm) {
    [pscustomobject]@{ pattern = $Pattern; edit_ppm = $EditPpm }
}

if ($Scale -eq 'smoke') {
    $sizes = @(100000, 1000000)
    $cases = @(
        New-Case 'steady_uniform' 0
        New-Case 'steady_uniform' 10000
        New-Case 'steady_uniform' 200000
        New-Case 'steady_uniform' 1000000
        New-Case 'steady_localized' 200000
        New-Case 'bursty_uniform' 10000
        New-Case 'phase' 200000
    )
} else {
    $sizes = @(100000, 1000000, 10000000, 100000000)
    $cases = @(
        New-Case 'steady_uniform' 0
        New-Case 'steady_uniform' 100
        New-Case 'steady_uniform' 1000
        New-Case 'steady_uniform' 10000
        New-Case 'steady_uniform' 50000
        New-Case 'steady_uniform' 200000
        New-Case 'steady_uniform' 500000
        New-Case 'steady_uniform' 1000000
        New-Case 'steady_localized' 1000
        New-Case 'steady_localized' 10000
        New-Case 'steady_localized' 200000
        New-Case 'steady_localized' 1000000
        New-Case 'bursty_uniform' 10000
        New-Case 'phase' 200000
    )
}

if ($OnlyHuge) {
    if ($Scale -ne 'full') { throw '-OnlyHuge requires -Scale full.' }
    $sizes = @(100000000)
}

function Get-TargetEdits([long]$N) {
    if ($N -le 1000000) { return 1024L }
    return 512L
}

function Get-Operations([long]$N, [int]$EditPpm) {
    if ($EditPpm -eq 0) {
        if ($N -le 1000000) { return 2000000L }
        if ($N -le 10000000) { return 1000000L }
        return 500000L
    }
    $operations = [long][math]::Ceiling(
        (Get-TargetEdits $N) * 1000000.0 / $EditPpm)
    return [math]::Min(8000000L, [math]::Max(1L, $operations))
}

function Get-Repeats([long]$N) {
    if ($N -le 1000000) { return 2 }
    return 1
}

$jobs = @()
foreach ($n in $sizes) {
    foreach ($case in $cases) {
        $operations = Get-Operations $n $case.edit_ppm
        $repeats = Get-Repeats $n
        foreach ($profile in $profileList) {
            $name = '{0}-n{1}-{2}-ppm{3}' -f `
                $profile, $n, $case.pattern, $case.edit_ppm
            $jobs += [pscustomobject]@{
                profile = $profile
                n = [long]$n
                pattern = $case.pattern
                edit_ppm = [int]$case.edit_ppm
                operations = [long]$operations
                repeats = [int]$repeats
                name = $name
                output = Join-Path $partials "$name.csv"
                summary = Join-Path $partials "$name-summary.csv"
                stdout = Join-Path $logs "$name.out.log"
                stderr = Join-Path $logs "$name.err.log"
            }
        }
    }
}

# Keep profiles of one cell adjacent, but interleave N=100m with smaller sizes.
# Twelve simultaneous huge adaptive rebuilds could consume almost all 32 GiB;
# this ordering still occupies all cores without deliberately forcing paging.
$jobs = @($jobs | Sort-Object pattern, edit_ppm,
    @{ Expression = 'n'; Descending = $true }, profile)

function Test-Partial($Job) {
    if (-not (Test-Path -LiteralPath $Job.output -PathType Leaf)) { return $false }
    try {
        $rows = @(Import-Csv -LiteralPath $Job.output)
        if ($rows.Count -ne 7 * $Job.repeats) { return $false }
        $expectedWorkload = "$($Job.pattern)_edit_ppm_$($Job.edit_ppm)"
        $invalidRows = @($rows | Where-Object {
            $_.status -ne 'ok' -or $_.profile -ne $Job.profile -or
            $_.suite -ne 'large' -or $_.type -ne 'uint32' -or
            [long]$_.n -ne $Job.n -or
            [long]$_.operations -ne $Job.operations -or
            $_.workload -ne $expectedWorkload
        })
        if ($invalidRows.Count -ne 0) {
            return $false
        }
        $expectedCandidates = @(
            'adaptive_shape_deferred', 'std::vector_reserved',
            'tiered_forced_leaf_64', 'tiered_forced_leaf_128',
            'tiered_forced_leaf_256', 'tiered_forced_leaf_512',
            'tiered_forced_leaf_1024'
        ) | Sort-Object
        for ($repeat = 0; $repeat -lt $Job.repeats; ++$repeat) {
            $repeatRows = @($rows | Where-Object { [int]$_.repeat -eq $repeat })
            if ($repeatRows.Count -ne 7) { return $false }
            $actualCandidates = @($repeatRows.structure | Sort-Object -Unique)
            if (($actualCandidates -join '|') -ne
                ($expectedCandidates -join '|')) { return $false }
            if (@($repeatRows.seed | Sort-Object -Unique).Count -ne 1 -or
                @($repeatRows.checksum | Sort-Object -Unique).Count -ne 1) {
                return $false
            }
        }
        return $true
    } catch {
        return $false
    }
}

function Get-TextSha256([string]$Text) {
    $algorithm = [Security.Cryptography.SHA256]::Create()
    try {
        return -join ($algorithm.ComputeHash([Text.Encoding]::UTF8.GetBytes($Text)) |
            ForEach-Object { $_.ToString('x2') })
    } finally {
        $algorithm.Dispose()
    }
}

$binaryContract = @($profileList | ForEach-Object {
    $binaryPath = Join-Path $projectRoot "out\bin\$_\Release\uc_bench.exe"
    [ordered]@{
        profile = $_
        sha256 = (Get-FileHash -LiteralPath $binaryPath -Algorithm SHA256).Hash.ToLowerInvariant()
    }
})
$contract = [ordered]@{
    version = 1
    scale = $Scale
    workers = $Workers
    max_huge_workers = $MaxHugeWorkers
    only_huge = [bool]$OnlyHuge
    profiles = $profileList
    binaries = $binaryContract
    jobs = @($jobs | Select-Object profile,n,pattern,edit_ppm,operations,repeats,name)
}
$planSignature = Get-TextSha256 ($contract | ConvertTo-Json -Depth 8 -Compress)
$planPath = Join-Path $result 'large-plan.json'
$isResume = Test-Path -LiteralPath $planPath -PathType Leaf
if ($isResume) {
    $existingPlan = Get-Content -Raw -LiteralPath $planPath | ConvertFrom-Json
    if (-not $existingPlan.plan_signature -or
        $existingPlan.plan_signature -ne $planSignature) {
        throw 'ResultDirectory belongs to a different plan or benchmark binary; choose a new directory.'
    }
}

if (-not $isResume) {
    & powershell.exe -NoProfile -ExecutionPolicy Bypass `
        -File 'tools\write_benchmark_environment.ps1' `
        -ResultDirectory $result -RunId $runId -Scale "large-$Scale-parallel" `
        -Profiles ($profileList -join ' ')
    if ($LASTEXITCODE -ne 0) { throw 'Could not write benchmark environment.' }
}

$plan = [ordered]@{
    run_id = $runId
    generated_at = (Get-Date).ToString('o')
    plan_signature = $planSignature
    workers = $Workers
    max_huge_workers = $MaxHugeWorkers
    only_huge = [bool]$OnlyHuge
    logical_processors = [Environment]::ProcessorCount
    affinity_policy = 'one distinct logical CPU per worker slot'
    profiles = $profileList
    binaries = $binaryContract
    candidates = @('std::vector_reserved', 'tiered_forced_leaf_64',
        'tiered_forced_leaf_128', 'tiered_forced_leaf_256',
        'tiered_forced_leaf_512', 'tiered_forced_leaf_1024',
        'adaptive_shape_deferred')
    sizes = $sizes
    cases = $cases
    operation_cap = 8000000
    target_edits = [ordered]@{ n_le_1m = 1024; n_10m = 512; n_100m = 512 }
    notes = @(
        'Cells run concurrently; candidates inside one cell run sequentially on one trace.',
        'Construction time is recorded separately and excluded from ns_per_op.',
        'Parallel memory-bandwidth contention is accepted for exploratory throughput.',
        'At most MaxHugeWorkers N=100m cells run together to avoid paging.',
        'Target edits are upper goals; the 8m operation cap can reduce very sparse cells.',
        'All seven candidates are mandatory; std::vector is never cost-guard skipped.'
    )
    jobs = @($jobs | Select-Object profile,n,pattern,edit_ppm,operations,repeats,name)
}
if (-not $isResume) {
    $plan | ConvertTo-Json -Depth 8 |
        Set-Content -LiteralPath $planPath -Encoding utf8
}

$completed = 0
$pending = [Collections.ArrayList]::new()
foreach ($job in $jobs) {
    if (Test-Partial $job) {
        ++$completed
    } else {
        [void]$pending.Add($job)
    }
}
$total = $jobs.Count
$running = @()
$logical = [Environment]::ProcessorCount
$cpuSlots = @(for ($slot = 0; $slot -lt $Workers; ++$slot) {
    [math]::Min($logical - 1,
        [int][math]::Floor(($slot + 0.5) * $logical / $Workers))
})

Write-Host ("Progress {0:N1}% ({1}/{2}); workers={3}; result={4}" -f `
    (100.0 * $completed / $total), $completed, $total, $Workers, $result)

try {
    while ($pending.Count -ne 0 -or $running.Count -ne 0) {
        while ($pending.Count -ne 0 -and $running.Count -lt $Workers) {
            $hugeRunning = @($running | Where-Object {
                $_.job.n -ge 100000000
            }).Count
            $nextIndex = -1
            for ($index = 0; $index -lt $pending.Count; ++$index) {
                if ($pending[$index].n -lt 100000000 -or
                    $hugeRunning -lt $MaxHugeWorkers) {
                    $nextIndex = $index
                    break
                }
            }
            if ($nextIndex -lt 0) { break }
            $job = $pending[$nextIndex]
            $pending.RemoveAt($nextIndex)
            $binary = Join-Path $projectRoot `
                "out\bin\$($job.profile)\Release\uc_bench.exe"
            $arguments = @(
                '--suite', 'large', '--scale', 'smoke',
                '--n', [string]$job.n,
                '--operations', [string]$job.operations,
                '--edit-ppm', [string]$job.edit_ppm,
                '--pattern', $job.pattern,
                '--leaf', '512', '--fanout', '64', '--levels', '4',
                '--repeats', [string]$job.repeats,
                '--output', $job.output,
                '--summary', $job.summary
            )
            # Start-Process enumerates the inherited environment and fails when
            # VS has supplied both Path and PATH.  ProcessStartInfo inherits the
            # environment directly and avoids that Windows PowerShell 5.1 bug.
            $quotedArguments = @($arguments | ForEach-Object {
                $argumentText = [string]$_
                if ($argumentText -match '[\s"]') {
                    '"' + ($argumentText -replace '"', '\"') + '"'
                } else {
                    $argumentText
                }
            }) -join ' '
            $startInfo = [Diagnostics.ProcessStartInfo]::new()
            $startInfo.FileName = $binary
            $startInfo.Arguments = $quotedArguments
            $startInfo.UseShellExecute = $false
            $startInfo.CreateNoWindow = $true
            $startInfo.RedirectStandardOutput = $true
            $startInfo.RedirectStandardError = $true
            $process = [Diagnostics.Process]::new()
            $process.StartInfo = $startInfo
            if (-not $process.Start()) {
                throw "Could not start large job: $($job.name)"
            }
            $usedSlots = @($running | ForEach-Object { $_.slot })
            $slot = 0
            while ($usedSlots -contains $slot) { ++$slot }
            try {
                $cpu = $cpuSlots[$slot]
                $process.ProcessorAffinity = [IntPtr](1L -shl $cpu)
            } catch {
                Add-Content -LiteralPath $job.stderr `
                    -Value "Affinity warning: $($_.Exception.Message)"
            }
            $running += [pscustomobject]@{
                process = $process
                job = $job
                slot = $slot
            }
        }

        Start-Sleep -Milliseconds 250
        $stillRunning = @()
        foreach ($entry in $running) {
            if (-not $entry.process.HasExited) {
                $stillRunning += $entry
                continue
            }
            $entry.process.WaitForExit()
            $exitCode = $entry.process.ExitCode
            $stdoutText = $entry.process.StandardOutput.ReadToEnd()
            $stderrText = $entry.process.StandardError.ReadToEnd()
            Set-Content -LiteralPath $entry.job.stdout -Value $stdoutText
            Set-Content -LiteralPath $entry.job.stderr -Value $stderrText
            $partialValid = Test-Partial $entry.job
            # Windows PowerShell may expose a null ExitCode for a very short
            # redirected process even after WaitForExit.  A fully validated
            # 7*candidate CSV is authoritative in that narrow case.
            if (($null -ne $exitCode -and $exitCode -ne 0) -or -not $partialValid) {
                throw "Large job failed: $($entry.job.name); exit=$exitCode; $stderrText"
            }
            ++$completed
            Write-Host ("Progress {0:N1}% ({1}/{2}) completed {3}" -f `
                (100.0 * $completed / $total), $completed, $total, $entry.job.name)
        }
        $running = $stillRunning
    }
} catch {
    foreach ($entry in $running) {
        if (-not $entry.process.HasExited) { $entry.process.Kill() }
    }
    throw
}

foreach ($job in $jobs) {
    if (-not (Test-Partial $job)) {
        throw "Partial became invalid before merge: $($job.name)"
    }
}
$rawRows = @($jobs | ForEach-Object {
    Import-Csv -LiteralPath $_.output
})
$raw = Join-Path $result 'large-raw.csv'
$rawRows | Export-Csv -LiteralPath $raw -NoTypeInformation -Encoding utf8

& powershell.exe -NoProfile -ExecutionPolicy Bypass `
    -File 'tools\analyze_results.ps1' -InputDirectory $result `
    -Pattern 'large-raw.csv' `
    -OutputFile (Join-Path $result 'aggregate-large.csv') `
    -CellMedianFile (Join-Path $result 'cell-medians-large.csv')
if ($LASTEXITCODE -ne 0) { throw 'Large result analysis failed.' }

& powershell.exe -NoProfile -ExecutionPolicy Bypass `
    -File 'tools\analyze_large_comparison.ps1' -InputDirectory $result
if ($LASTEXITCODE -ne 0) { throw 'Large comparison analysis failed.' }

@(
    'powershell.exe -NoProfile -ExecutionPolicy Bypass -File tools\analyze_results.ps1 -InputDirectory <run> -Pattern large-raw.csv -OutputFile <run>\aggregate-large.csv -CellMedianFile <run>\cell-medians-large.csv',
    'powershell.exe -NoProfile -ExecutionPolicy Bypass -File tools\analyze_large_comparison.ps1 -InputDirectory <run>'
) | Set-Content -LiteralPath (Join-Path $result 'analysis-command.txt') -Encoding ascii

$manifestPath = Join-Path $result 'results-manifest.sha256'
$manifestLines = @(Get-ChildItem -LiteralPath $result -Recurse -File |
    Where-Object { $_.FullName -ne $manifestPath } |
    Sort-Object FullName | ForEach-Object {
        $relative = $_.FullName.Substring($result.Length).TrimStart('\') -replace '\\', '/'
        $hash = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
        "$hash  $relative"
    })
$manifestLines | Set-Content -LiteralPath $manifestPath -Encoding ascii

Write-Host "Progress 100.0% ($total/$total). Large benchmark completed: $result"