Experiments
This commit is contained in:
Efim Beshmenev
@@ -0,0 +1,414 @@
|
||||
param(
|
||||
[ValidateSet('smoke', 'full')]
|
||||
[string]$Scale = 'full',
|
||||
|
||||
[ValidateRange(2, 12)]
|
||||
[int]$Workers = 12,
|
||||
|
||||
[ValidateRange(1, 12)]
|
||||
[int]$MaxHugeWorkers = 3,
|
||||
|
||||
[switch]$OnlyHuge,
|
||||
|
||||
[string]$ResultDirectory = '',
|
||||
|
||||
[string]$Profiles = 'scalar baseline avx2'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$projectRoot = (Resolve-Path '.').Path
|
||||
if (-not $ResultDirectory) {
|
||||
$runId = "$(Get-Date -Format yyyyMMdd-HHmmss)-large-$Scale"
|
||||
$ResultDirectory = Join-Path 'results\benchmarks' $runId
|
||||
}
|
||||
$result = if ([IO.Path]::IsPathRooted($ResultDirectory)) {
|
||||
[IO.Path]::GetFullPath($ResultDirectory)
|
||||
} else {
|
||||
[IO.Path]::GetFullPath((Join-Path $projectRoot $ResultDirectory))
|
||||
}
|
||||
$runId = Split-Path -Leaf $result
|
||||
$partials = Join-Path $result 'partials'
|
||||
$logs = Join-Path $result 'logs'
|
||||
New-Item -ItemType Directory -Force -Path $partials, $logs | Out-Null
|
||||
|
||||
$profileList = @($Profiles -split '[, ]+' | Where-Object { $_ })
|
||||
if ($profileList.Count -eq 0) { throw 'At least one profile is required.' }
|
||||
foreach ($profile in $profileList) {
|
||||
$binary = Join-Path $projectRoot "out\bin\$profile\Release\uc_bench.exe"
|
||||
if (-not (Test-Path -LiteralPath $binary -PathType Leaf)) {
|
||||
throw "Missing benchmark binary for profile '$profile': $binary"
|
||||
}
|
||||
}
|
||||
|
||||
function New-Case([string]$Pattern, [int]$EditPpm) {
|
||||
[pscustomobject]@{ pattern = $Pattern; edit_ppm = $EditPpm }
|
||||
}
|
||||
|
||||
if ($Scale -eq 'smoke') {
|
||||
$sizes = @(100000, 1000000)
|
||||
$cases = @(
|
||||
New-Case 'steady_uniform' 0
|
||||
New-Case 'steady_uniform' 10000
|
||||
New-Case 'steady_uniform' 200000
|
||||
New-Case 'steady_uniform' 1000000
|
||||
New-Case 'steady_localized' 200000
|
||||
New-Case 'bursty_uniform' 10000
|
||||
New-Case 'phase' 200000
|
||||
)
|
||||
} else {
|
||||
$sizes = @(100000, 1000000, 10000000, 100000000)
|
||||
$cases = @(
|
||||
New-Case 'steady_uniform' 0
|
||||
New-Case 'steady_uniform' 100
|
||||
New-Case 'steady_uniform' 1000
|
||||
New-Case 'steady_uniform' 10000
|
||||
New-Case 'steady_uniform' 50000
|
||||
New-Case 'steady_uniform' 200000
|
||||
New-Case 'steady_uniform' 500000
|
||||
New-Case 'steady_uniform' 1000000
|
||||
New-Case 'steady_localized' 1000
|
||||
New-Case 'steady_localized' 10000
|
||||
New-Case 'steady_localized' 200000
|
||||
New-Case 'steady_localized' 1000000
|
||||
New-Case 'bursty_uniform' 10000
|
||||
New-Case 'phase' 200000
|
||||
)
|
||||
}
|
||||
|
||||
if ($OnlyHuge) {
|
||||
if ($Scale -ne 'full') { throw '-OnlyHuge requires -Scale full.' }
|
||||
$sizes = @(100000000)
|
||||
}
|
||||
|
||||
function Get-TargetEdits([long]$N) {
|
||||
if ($N -le 1000000) { return 1024L }
|
||||
return 512L
|
||||
}
|
||||
|
||||
function Get-Operations([long]$N, [int]$EditPpm) {
|
||||
if ($EditPpm -eq 0) {
|
||||
if ($N -le 1000000) { return 2000000L }
|
||||
if ($N -le 10000000) { return 1000000L }
|
||||
return 500000L
|
||||
}
|
||||
$operations = [long][math]::Ceiling(
|
||||
(Get-TargetEdits $N) * 1000000.0 / $EditPpm)
|
||||
return [math]::Min(8000000L, [math]::Max(1L, $operations))
|
||||
}
|
||||
|
||||
function Get-Repeats([long]$N) {
|
||||
if ($N -le 1000000) { return 2 }
|
||||
return 1
|
||||
}
|
||||
|
||||
$jobs = @()
|
||||
foreach ($n in $sizes) {
|
||||
foreach ($case in $cases) {
|
||||
$operations = Get-Operations $n $case.edit_ppm
|
||||
$repeats = Get-Repeats $n
|
||||
foreach ($profile in $profileList) {
|
||||
$name = '{0}-n{1}-{2}-ppm{3}' -f `
|
||||
$profile, $n, $case.pattern, $case.edit_ppm
|
||||
$jobs += [pscustomobject]@{
|
||||
profile = $profile
|
||||
n = [long]$n
|
||||
pattern = $case.pattern
|
||||
edit_ppm = [int]$case.edit_ppm
|
||||
operations = [long]$operations
|
||||
repeats = [int]$repeats
|
||||
name = $name
|
||||
output = Join-Path $partials "$name.csv"
|
||||
summary = Join-Path $partials "$name-summary.csv"
|
||||
stdout = Join-Path $logs "$name.out.log"
|
||||
stderr = Join-Path $logs "$name.err.log"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Keep profiles of one cell adjacent, but interleave N=100m with smaller sizes.
|
||||
# Twelve simultaneous huge adaptive rebuilds could consume almost all 32 GiB;
|
||||
# this ordering still occupies all cores without deliberately forcing paging.
|
||||
$jobs = @($jobs | Sort-Object pattern, edit_ppm,
|
||||
@{ Expression = 'n'; Descending = $true }, profile)
|
||||
|
||||
function Test-Partial($Job) {
|
||||
if (-not (Test-Path -LiteralPath $Job.output -PathType Leaf)) { return $false }
|
||||
try {
|
||||
$rows = @(Import-Csv -LiteralPath $Job.output)
|
||||
if ($rows.Count -ne 7 * $Job.repeats) { return $false }
|
||||
$expectedWorkload = "$($Job.pattern)_edit_ppm_$($Job.edit_ppm)"
|
||||
$invalidRows = @($rows | Where-Object {
|
||||
$_.status -ne 'ok' -or $_.profile -ne $Job.profile -or
|
||||
$_.suite -ne 'large' -or $_.type -ne 'uint32' -or
|
||||
[long]$_.n -ne $Job.n -or
|
||||
[long]$_.operations -ne $Job.operations -or
|
||||
$_.workload -ne $expectedWorkload
|
||||
})
|
||||
if ($invalidRows.Count -ne 0) {
|
||||
return $false
|
||||
}
|
||||
$expectedCandidates = @(
|
||||
'adaptive_shape_deferred', 'std::vector_reserved',
|
||||
'tiered_forced_leaf_64', 'tiered_forced_leaf_128',
|
||||
'tiered_forced_leaf_256', 'tiered_forced_leaf_512',
|
||||
'tiered_forced_leaf_1024'
|
||||
) | Sort-Object
|
||||
for ($repeat = 0; $repeat -lt $Job.repeats; ++$repeat) {
|
||||
$repeatRows = @($rows | Where-Object { [int]$_.repeat -eq $repeat })
|
||||
if ($repeatRows.Count -ne 7) { return $false }
|
||||
$actualCandidates = @($repeatRows.structure | Sort-Object -Unique)
|
||||
if (($actualCandidates -join '|') -ne
|
||||
($expectedCandidates -join '|')) { return $false }
|
||||
if (@($repeatRows.seed | Sort-Object -Unique).Count -ne 1 -or
|
||||
@($repeatRows.checksum | Sort-Object -Unique).Count -ne 1) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Get-TextSha256([string]$Text) {
|
||||
$algorithm = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return -join ($algorithm.ComputeHash([Text.Encoding]::UTF8.GetBytes($Text)) |
|
||||
ForEach-Object { $_.ToString('x2') })
|
||||
} finally {
|
||||
$algorithm.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
$binaryContract = @($profileList | ForEach-Object {
|
||||
$binaryPath = Join-Path $projectRoot "out\bin\$_\Release\uc_bench.exe"
|
||||
[ordered]@{
|
||||
profile = $_
|
||||
sha256 = (Get-FileHash -LiteralPath $binaryPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
})
|
||||
$contract = [ordered]@{
|
||||
version = 1
|
||||
scale = $Scale
|
||||
workers = $Workers
|
||||
max_huge_workers = $MaxHugeWorkers
|
||||
only_huge = [bool]$OnlyHuge
|
||||
profiles = $profileList
|
||||
binaries = $binaryContract
|
||||
jobs = @($jobs | Select-Object profile,n,pattern,edit_ppm,operations,repeats,name)
|
||||
}
|
||||
$planSignature = Get-TextSha256 ($contract | ConvertTo-Json -Depth 8 -Compress)
|
||||
$planPath = Join-Path $result 'large-plan.json'
|
||||
$isResume = Test-Path -LiteralPath $planPath -PathType Leaf
|
||||
if ($isResume) {
|
||||
$existingPlan = Get-Content -Raw -LiteralPath $planPath | ConvertFrom-Json
|
||||
if (-not $existingPlan.plan_signature -or
|
||||
$existingPlan.plan_signature -ne $planSignature) {
|
||||
throw 'ResultDirectory belongs to a different plan or benchmark binary; choose a new directory.'
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $isResume) {
|
||||
& powershell.exe -NoProfile -ExecutionPolicy Bypass `
|
||||
-File 'tools\write_benchmark_environment.ps1' `
|
||||
-ResultDirectory $result -RunId $runId -Scale "large-$Scale-parallel" `
|
||||
-Profiles ($profileList -join ' ')
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Could not write benchmark environment.' }
|
||||
}
|
||||
|
||||
$plan = [ordered]@{
|
||||
run_id = $runId
|
||||
generated_at = (Get-Date).ToString('o')
|
||||
plan_signature = $planSignature
|
||||
workers = $Workers
|
||||
max_huge_workers = $MaxHugeWorkers
|
||||
only_huge = [bool]$OnlyHuge
|
||||
logical_processors = [Environment]::ProcessorCount
|
||||
affinity_policy = 'one distinct logical CPU per worker slot'
|
||||
profiles = $profileList
|
||||
binaries = $binaryContract
|
||||
candidates = @('std::vector_reserved', 'tiered_forced_leaf_64',
|
||||
'tiered_forced_leaf_128', 'tiered_forced_leaf_256',
|
||||
'tiered_forced_leaf_512', 'tiered_forced_leaf_1024',
|
||||
'adaptive_shape_deferred')
|
||||
sizes = $sizes
|
||||
cases = $cases
|
||||
operation_cap = 8000000
|
||||
target_edits = [ordered]@{ n_le_1m = 1024; n_10m = 512; n_100m = 512 }
|
||||
notes = @(
|
||||
'Cells run concurrently; candidates inside one cell run sequentially on one trace.',
|
||||
'Construction time is recorded separately and excluded from ns_per_op.',
|
||||
'Parallel memory-bandwidth contention is accepted for exploratory throughput.',
|
||||
'At most MaxHugeWorkers N=100m cells run together to avoid paging.',
|
||||
'Target edits are upper goals; the 8m operation cap can reduce very sparse cells.',
|
||||
'All seven candidates are mandatory; std::vector is never cost-guard skipped.'
|
||||
)
|
||||
jobs = @($jobs | Select-Object profile,n,pattern,edit_ppm,operations,repeats,name)
|
||||
}
|
||||
if (-not $isResume) {
|
||||
$plan | ConvertTo-Json -Depth 8 |
|
||||
Set-Content -LiteralPath $planPath -Encoding utf8
|
||||
}
|
||||
|
||||
$completed = 0
|
||||
$pending = [Collections.ArrayList]::new()
|
||||
foreach ($job in $jobs) {
|
||||
if (Test-Partial $job) {
|
||||
++$completed
|
||||
} else {
|
||||
[void]$pending.Add($job)
|
||||
}
|
||||
}
|
||||
$total = $jobs.Count
|
||||
$running = @()
|
||||
$logical = [Environment]::ProcessorCount
|
||||
$cpuSlots = @(for ($slot = 0; $slot -lt $Workers; ++$slot) {
|
||||
[math]::Min($logical - 1,
|
||||
[int][math]::Floor(($slot + 0.5) * $logical / $Workers))
|
||||
})
|
||||
|
||||
Write-Host ("Progress {0:N1}% ({1}/{2}); workers={3}; result={4}" -f `
|
||||
(100.0 * $completed / $total), $completed, $total, $Workers, $result)
|
||||
|
||||
try {
|
||||
while ($pending.Count -ne 0 -or $running.Count -ne 0) {
|
||||
while ($pending.Count -ne 0 -and $running.Count -lt $Workers) {
|
||||
$hugeRunning = @($running | Where-Object {
|
||||
$_.job.n -ge 100000000
|
||||
}).Count
|
||||
$nextIndex = -1
|
||||
for ($index = 0; $index -lt $pending.Count; ++$index) {
|
||||
if ($pending[$index].n -lt 100000000 -or
|
||||
$hugeRunning -lt $MaxHugeWorkers) {
|
||||
$nextIndex = $index
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($nextIndex -lt 0) { break }
|
||||
$job = $pending[$nextIndex]
|
||||
$pending.RemoveAt($nextIndex)
|
||||
$binary = Join-Path $projectRoot `
|
||||
"out\bin\$($job.profile)\Release\uc_bench.exe"
|
||||
$arguments = @(
|
||||
'--suite', 'large', '--scale', 'smoke',
|
||||
'--n', [string]$job.n,
|
||||
'--operations', [string]$job.operations,
|
||||
'--edit-ppm', [string]$job.edit_ppm,
|
||||
'--pattern', $job.pattern,
|
||||
'--leaf', '512', '--fanout', '64', '--levels', '4',
|
||||
'--repeats', [string]$job.repeats,
|
||||
'--output', $job.output,
|
||||
'--summary', $job.summary
|
||||
)
|
||||
# Start-Process enumerates the inherited environment and fails when
|
||||
# VS has supplied both Path and PATH. ProcessStartInfo inherits the
|
||||
# environment directly and avoids that Windows PowerShell 5.1 bug.
|
||||
$quotedArguments = @($arguments | ForEach-Object {
|
||||
$argumentText = [string]$_
|
||||
if ($argumentText -match '[\s"]') {
|
||||
'"' + ($argumentText -replace '"', '\"') + '"'
|
||||
} else {
|
||||
$argumentText
|
||||
}
|
||||
}) -join ' '
|
||||
$startInfo = [Diagnostics.ProcessStartInfo]::new()
|
||||
$startInfo.FileName = $binary
|
||||
$startInfo.Arguments = $quotedArguments
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.CreateNoWindow = $true
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$process = [Diagnostics.Process]::new()
|
||||
$process.StartInfo = $startInfo
|
||||
if (-not $process.Start()) {
|
||||
throw "Could not start large job: $($job.name)"
|
||||
}
|
||||
$usedSlots = @($running | ForEach-Object { $_.slot })
|
||||
$slot = 0
|
||||
while ($usedSlots -contains $slot) { ++$slot }
|
||||
try {
|
||||
$cpu = $cpuSlots[$slot]
|
||||
$process.ProcessorAffinity = [IntPtr](1L -shl $cpu)
|
||||
} catch {
|
||||
Add-Content -LiteralPath $job.stderr `
|
||||
-Value "Affinity warning: $($_.Exception.Message)"
|
||||
}
|
||||
$running += [pscustomobject]@{
|
||||
process = $process
|
||||
job = $job
|
||||
slot = $slot
|
||||
}
|
||||
}
|
||||
|
||||
Start-Sleep -Milliseconds 250
|
||||
$stillRunning = @()
|
||||
foreach ($entry in $running) {
|
||||
if (-not $entry.process.HasExited) {
|
||||
$stillRunning += $entry
|
||||
continue
|
||||
}
|
||||
$entry.process.WaitForExit()
|
||||
$exitCode = $entry.process.ExitCode
|
||||
$stdoutText = $entry.process.StandardOutput.ReadToEnd()
|
||||
$stderrText = $entry.process.StandardError.ReadToEnd()
|
||||
Set-Content -LiteralPath $entry.job.stdout -Value $stdoutText
|
||||
Set-Content -LiteralPath $entry.job.stderr -Value $stderrText
|
||||
$partialValid = Test-Partial $entry.job
|
||||
# Windows PowerShell may expose a null ExitCode for a very short
|
||||
# redirected process even after WaitForExit. A fully validated
|
||||
# 7*candidate CSV is authoritative in that narrow case.
|
||||
if (($null -ne $exitCode -and $exitCode -ne 0) -or -not $partialValid) {
|
||||
throw "Large job failed: $($entry.job.name); exit=$exitCode; $stderrText"
|
||||
}
|
||||
++$completed
|
||||
Write-Host ("Progress {0:N1}% ({1}/{2}) completed {3}" -f `
|
||||
(100.0 * $completed / $total), $completed, $total, $entry.job.name)
|
||||
}
|
||||
$running = $stillRunning
|
||||
}
|
||||
} catch {
|
||||
foreach ($entry in $running) {
|
||||
if (-not $entry.process.HasExited) { $entry.process.Kill() }
|
||||
}
|
||||
throw
|
||||
}
|
||||
|
||||
foreach ($job in $jobs) {
|
||||
if (-not (Test-Partial $job)) {
|
||||
throw "Partial became invalid before merge: $($job.name)"
|
||||
}
|
||||
}
|
||||
$rawRows = @($jobs | ForEach-Object {
|
||||
Import-Csv -LiteralPath $_.output
|
||||
})
|
||||
$raw = Join-Path $result 'large-raw.csv'
|
||||
$rawRows | Export-Csv -LiteralPath $raw -NoTypeInformation -Encoding utf8
|
||||
|
||||
& powershell.exe -NoProfile -ExecutionPolicy Bypass `
|
||||
-File 'tools\analyze_results.ps1' -InputDirectory $result `
|
||||
-Pattern 'large-raw.csv' `
|
||||
-OutputFile (Join-Path $result 'aggregate-large.csv') `
|
||||
-CellMedianFile (Join-Path $result 'cell-medians-large.csv')
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Large result analysis failed.' }
|
||||
|
||||
& powershell.exe -NoProfile -ExecutionPolicy Bypass `
|
||||
-File 'tools\analyze_large_comparison.ps1' -InputDirectory $result
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Large comparison analysis failed.' }
|
||||
|
||||
@(
|
||||
'powershell.exe -NoProfile -ExecutionPolicy Bypass -File tools\analyze_results.ps1 -InputDirectory <run> -Pattern large-raw.csv -OutputFile <run>\aggregate-large.csv -CellMedianFile <run>\cell-medians-large.csv',
|
||||
'powershell.exe -NoProfile -ExecutionPolicy Bypass -File tools\analyze_large_comparison.ps1 -InputDirectory <run>'
|
||||
) | Set-Content -LiteralPath (Join-Path $result 'analysis-command.txt') -Encoding ascii
|
||||
|
||||
$manifestPath = Join-Path $result 'results-manifest.sha256'
|
||||
$manifestLines = @(Get-ChildItem -LiteralPath $result -Recurse -File |
|
||||
Where-Object { $_.FullName -ne $manifestPath } |
|
||||
Sort-Object FullName | ForEach-Object {
|
||||
$relative = $_.FullName.Substring($result.Length).TrimStart('\') -replace '\\', '/'
|
||||
$hash = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
"$hash $relative"
|
||||
})
|
||||
$manifestLines | Set-Content -LiteralPath $manifestPath -Encoding ascii
|
||||
|
||||
Write-Host "Progress 100.0% ($total/$total). Large benchmark completed: $result"
|
||||
Reference in New Issue
Block a user