Efim Beshmenev
187 lines
8.0 KiB
PowerShell
187 lines
8.0 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$InputDirectory,
|
|
|
|
[string]$OutputFile = "",
|
|
|
|
[string]$Pattern = "*-compare.csv",
|
|
|
|
[string]$CellMedianFile = ""
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$root = (Resolve-Path -LiteralPath $InputDirectory).Path
|
|
if (-not $OutputFile) {
|
|
$OutputFile = Join-Path $root 'aggregate.csv'
|
|
}
|
|
if (-not $CellMedianFile) {
|
|
$CellMedianFile = Join-Path $root 'cell-medians.csv'
|
|
}
|
|
|
|
$files = @(Get-ChildItem -LiteralPath $root -Filter $Pattern -File)
|
|
if ($files.Count -eq 0) {
|
|
throw "No $Pattern files found in $root"
|
|
}
|
|
|
|
$rows = @($files | ForEach-Object { Import-Csv -LiteralPath $_.FullName }) |
|
|
Where-Object { $_.status -eq 'ok' }
|
|
|
|
function Get-Median([double[]]$Values) {
|
|
if ($Values.Count -eq 0) { return [double]::NaN }
|
|
$ordered = @($Values | Sort-Object)
|
|
$middle = [int]($ordered.Count / 2)
|
|
if (($ordered.Count % 2) -eq 0) {
|
|
return ($ordered[$middle - 1] + $ordered[$middle]) / 2.0
|
|
}
|
|
return $ordered[$middle]
|
|
}
|
|
|
|
function Get-GeometricMean([double[]]$Values) {
|
|
$positive = @($Values | Where-Object { $_ -gt 0 -and -not [double]::IsNaN($_) })
|
|
if ($positive.Count -eq 0) { return [double]::NaN }
|
|
$logSum = 0.0
|
|
foreach ($value in $positive) { $logSum += [math]::Log($value) }
|
|
return [math]::Exp($logSum / $positive.Count)
|
|
}
|
|
|
|
function Get-Mode($Values) {
|
|
$groups = @($Values | Group-Object | Sort-Object Count -Descending)
|
|
if ($groups.Count -eq 0) { return '' }
|
|
return $groups[0].Name
|
|
}
|
|
|
|
# Every implementation must produce the same observable result for an
|
|
# identical trace. A mismatch invalidates the performance run.
|
|
$mismatches = @($rows |
|
|
Group-Object profile,type,workload,n,repeat,seed |
|
|
Where-Object { @($_.Group.checksum | Select-Object -Unique).Count -ne 1 })
|
|
if ($mismatches.Count -ne 0) {
|
|
throw "Checksum mismatch in $($mismatches.Count) benchmark cell(s)"
|
|
}
|
|
|
|
$medians = @($rows |
|
|
Group-Object profile,type,structure,workload,n,leaf,fanout,levels |
|
|
ForEach-Object {
|
|
$group = $_.Group
|
|
$candidate = $group[0].structure
|
|
if ($candidate -like 'adaptive_*' -or $candidate -like 'tiered_forced_*') {
|
|
$candidate += "@initial_$($group[0].leaf)_$($group[0].fanout)_$($group[0].levels)"
|
|
}
|
|
[pscustomobject]@{
|
|
profile = $group[0].profile
|
|
type = $group[0].type
|
|
structure = $candidate
|
|
workload = $group[0].workload
|
|
n = [uint64]$group[0].n
|
|
leaf = [uint64]$group[0].leaf
|
|
fanout = [uint64]$group[0].fanout
|
|
levels = [uint64]$group[0].levels
|
|
final_leaf = Get-Mode @($group.final_leaf)
|
|
final_fanout = Get-Mode @($group.final_fanout)
|
|
final_levels = Get-Mode @($group.final_levels)
|
|
final_mode = Get-Mode @($group.final_mode)
|
|
median_ns = Get-Median @($group.total_ns | ForEach-Object { [double]$_ })
|
|
median_ns_per_op = Get-Median @($group.ns_per_op | ForEach-Object { [double]$_ })
|
|
median_bytes = Get-Median @($group.allocated_bytes | ForEach-Object { [double]$_ })
|
|
median_p99_ns = Get-Median @($group.p99_ns | ForEach-Object { [double]$_ })
|
|
median_max_maintenance_ns = Get-Median @($group.max_maintenance_ns | ForEach-Object { [double]$_ })
|
|
median_max_transition_ns = Get-Median @($group.max_transition_ns | ForEach-Object { [double]$_ })
|
|
median_switches = Get-Median @($group.switches | ForEach-Object { [double]$_ })
|
|
median_shape_rebuilds = Get-Median @($group.shape_rebuilds | ForEach-Object { [double]$_ })
|
|
median_leaf_rebuilds = Get-Median @($group.leaf_rebuilds | ForEach-Object { [double]$_ })
|
|
median_directory_rebuilds = Get-Median @($group.directory_rebuilds | ForEach-Object { [double]$_ })
|
|
median_policy_evaluations = Get-Median @($group.policy_evaluations | ForEach-Object { [double]$_ })
|
|
median_last_forecast_operations = Get-Median @($group.last_forecast_operations | ForEach-Object { [double]$_ })
|
|
median_last_evidence_windows = Get-Median @($group.last_evidence_windows | ForEach-Object { [double]$_ })
|
|
median_last_vector_cost = Get-Median @($group.last_vector_cost | ForEach-Object { [double]$_ })
|
|
median_last_tiered_cost = Get-Median @($group.last_tiered_cost | ForEach-Object { [double]$_ })
|
|
median_last_current_cost = Get-Median @($group.last_current_cost | ForEach-Object { [double]$_ })
|
|
median_last_conversion_cost = Get-Median @($group.last_conversion_cost | ForEach-Object { [double]$_ })
|
|
median_last_expected_saving = Get-Median @($group.last_expected_saving | ForEach-Object { [double]$_ })
|
|
median_last_local_edit_fraction = Get-Median @($group.last_local_edit_fraction | ForEach-Object { [double]$_ })
|
|
median_last_edit_fraction = Get-Median @($group.last_edit_fraction | ForEach-Object { [double]$_ })
|
|
}
|
|
})
|
|
|
|
$medians | Sort-Object profile,type,workload,n,structure |
|
|
Export-Csv -LiteralPath $CellMedianFile -NoTypeInformation -Encoding utf8
|
|
|
|
$fixedNames = @('std::vector_reserved', 'std::deque',
|
|
'std::list_index_semantics')
|
|
$bestFixed = @{}
|
|
$medians | Group-Object profile,type,workload,n | ForEach-Object {
|
|
$eligible = @($_.Group | Where-Object {
|
|
$fixedNames -contains $_.structure -or $_.structure -like 'tiered_forced_leaf_*'
|
|
})
|
|
if ($eligible.Count -gt 0) {
|
|
$key = "$($_.Group[0].profile)|$($_.Group[0].type)|$($_.Group[0].workload)|$($_.Group[0].n)"
|
|
$bestFixed[$key] = ($eligible.median_ns | Measure-Object -Minimum).Minimum
|
|
}
|
|
}
|
|
|
|
$cellScores = @($medians | ForEach-Object {
|
|
$key = "$($_.profile)|$($_.type)|$($_.workload)|$($_.n)"
|
|
if ($bestFixed.ContainsKey($key) -and $_.median_ns -gt 0) {
|
|
[pscustomobject]@{
|
|
profile = $_.profile
|
|
structure = $_.structure
|
|
type = $_.type
|
|
workload = $_.workload
|
|
n = $_.n
|
|
score = [double]$bestFixed[$key] / [double]$_.median_ns
|
|
}
|
|
}
|
|
})
|
|
|
|
# Hierarchical aggregation prevents a family with more sizes from receiving a
|
|
# larger implicit weight: sizes -> workloads -> types -> compiler profiles.
|
|
$byWorkload = @($cellScores | Group-Object profile,structure,type,workload |
|
|
ForEach-Object {
|
|
$g = $_.Group
|
|
[pscustomobject]@{
|
|
profile = $g[0].profile; structure = $g[0].structure
|
|
type = $g[0].type; workload = $g[0].workload
|
|
score = Get-GeometricMean @($g.score)
|
|
}
|
|
})
|
|
$byType = @($byWorkload | Group-Object profile,structure,type |
|
|
ForEach-Object {
|
|
$g = $_.Group
|
|
[pscustomobject]@{
|
|
profile = $g[0].profile; structure = $g[0].structure
|
|
type = $g[0].type; score = Get-GeometricMean @($g.score)
|
|
}
|
|
})
|
|
$byProfile = @($byType | Group-Object profile,structure |
|
|
ForEach-Object {
|
|
$g = $_.Group
|
|
[pscustomobject]@{
|
|
profile = $g[0].profile
|
|
structure = $g[0].structure
|
|
score = Get-GeometricMean @($g.score)
|
|
types = $g.Count
|
|
cells = @($cellScores | Where-Object {
|
|
$_.profile -eq $g[0].profile -and $_.structure -eq $g[0].structure
|
|
}).Count
|
|
}
|
|
})
|
|
$overall = @($byProfile | Group-Object structure | ForEach-Object {
|
|
$g = $_.Group
|
|
[pscustomobject]@{
|
|
profile = 'ALL'
|
|
structure = $g[0].structure
|
|
score = Get-GeometricMean @($g.score)
|
|
types = ($g.types | Measure-Object -Maximum).Maximum
|
|
cells = ($g.cells | Measure-Object -Sum).Sum
|
|
}
|
|
})
|
|
|
|
@($byProfile + $overall) | Sort-Object profile,@{Expression='score';Descending=$true} |
|
|
Export-Csv -LiteralPath $OutputFile -NoTypeInformation -Encoding utf8
|
|
|
|
Write-Host "Validated $($rows.Count) raw rows from $($files.Count) profile file(s)."
|
|
Write-Host "Cell medians: $CellMedianFile"
|
|
Write-Host "Aggregate: $OutputFile"
|
|
@($overall | Sort-Object score -Descending) |
|
|
Format-Table structure,score,types,cells -AutoSize
|