param( [Parameter(Mandatory = $true)] [string]$InputDirectory, [string]$OutputFile = "", [string]$Pattern = "*-compare.csv", [string]$CellMedianFile = "" ) $ErrorActionPreference = 'Stop' $root = (Resolve-Path -LiteralPath $InputDirectory).Path if (-not $OutputFile) { $OutputFile = Join-Path $root 'aggregate.csv' } if (-not $CellMedianFile) { $CellMedianFile = Join-Path $root 'cell-medians.csv' } $files = @(Get-ChildItem -LiteralPath $root -Filter $Pattern -File) if ($files.Count -eq 0) { throw "No $Pattern files found in $root" } $rows = @($files | ForEach-Object { Import-Csv -LiteralPath $_.FullName }) | Where-Object { $_.status -eq 'ok' } function Get-Median([double[]]$Values) { if ($Values.Count -eq 0) { return [double]::NaN } $ordered = @($Values | Sort-Object) $middle = [int]($ordered.Count / 2) if (($ordered.Count % 2) -eq 0) { return ($ordered[$middle - 1] + $ordered[$middle]) / 2.0 } return $ordered[$middle] } function Get-GeometricMean([double[]]$Values) { $positive = @($Values | Where-Object { $_ -gt 0 -and -not [double]::IsNaN($_) }) if ($positive.Count -eq 0) { return [double]::NaN } $logSum = 0.0 foreach ($value in $positive) { $logSum += [math]::Log($value) } return [math]::Exp($logSum / $positive.Count) } function Get-Mode($Values) { $groups = @($Values | Group-Object | Sort-Object Count -Descending) if ($groups.Count -eq 0) { return '' } return $groups[0].Name } # Every implementation must produce the same observable result for an # identical trace. A mismatch invalidates the performance run. $mismatches = @($rows | Group-Object profile,type,workload,n,repeat,seed | Where-Object { @($_.Group.checksum | Select-Object -Unique).Count -ne 1 }) if ($mismatches.Count -ne 0) { throw "Checksum mismatch in $($mismatches.Count) benchmark cell(s)" } $medians = @($rows | Group-Object profile,type,structure,workload,n,leaf,fanout,levels | ForEach-Object { $group = $_.Group $candidate = $group[0].structure if ($candidate -like 'adaptive_*' -or $candidate -like 'tiered_forced_*') { $candidate += "@initial_$($group[0].leaf)_$($group[0].fanout)_$($group[0].levels)" } [pscustomobject]@{ profile = $group[0].profile type = $group[0].type structure = $candidate workload = $group[0].workload n = [uint64]$group[0].n leaf = [uint64]$group[0].leaf fanout = [uint64]$group[0].fanout levels = [uint64]$group[0].levels final_leaf = Get-Mode @($group.final_leaf) final_fanout = Get-Mode @($group.final_fanout) final_levels = Get-Mode @($group.final_levels) final_mode = Get-Mode @($group.final_mode) median_ns = Get-Median @($group.total_ns | ForEach-Object { [double]$_ }) median_ns_per_op = Get-Median @($group.ns_per_op | ForEach-Object { [double]$_ }) median_bytes = Get-Median @($group.allocated_bytes | ForEach-Object { [double]$_ }) median_p99_ns = Get-Median @($group.p99_ns | ForEach-Object { [double]$_ }) median_max_maintenance_ns = Get-Median @($group.max_maintenance_ns | ForEach-Object { [double]$_ }) median_max_transition_ns = Get-Median @($group.max_transition_ns | ForEach-Object { [double]$_ }) median_switches = Get-Median @($group.switches | ForEach-Object { [double]$_ }) median_shape_rebuilds = Get-Median @($group.shape_rebuilds | ForEach-Object { [double]$_ }) median_leaf_rebuilds = Get-Median @($group.leaf_rebuilds | ForEach-Object { [double]$_ }) median_directory_rebuilds = Get-Median @($group.directory_rebuilds | ForEach-Object { [double]$_ }) median_policy_evaluations = Get-Median @($group.policy_evaluations | ForEach-Object { [double]$_ }) median_last_forecast_operations = Get-Median @($group.last_forecast_operations | ForEach-Object { [double]$_ }) median_last_evidence_windows = Get-Median @($group.last_evidence_windows | ForEach-Object { [double]$_ }) median_last_vector_cost = Get-Median @($group.last_vector_cost | ForEach-Object { [double]$_ }) median_last_tiered_cost = Get-Median @($group.last_tiered_cost | ForEach-Object { [double]$_ }) median_last_current_cost = Get-Median @($group.last_current_cost | ForEach-Object { [double]$_ }) median_last_conversion_cost = Get-Median @($group.last_conversion_cost | ForEach-Object { [double]$_ }) median_last_expected_saving = Get-Median @($group.last_expected_saving | ForEach-Object { [double]$_ }) median_last_local_edit_fraction = Get-Median @($group.last_local_edit_fraction | ForEach-Object { [double]$_ }) median_last_edit_fraction = Get-Median @($group.last_edit_fraction | ForEach-Object { [double]$_ }) } }) $medians | Sort-Object profile,type,workload,n,structure | Export-Csv -LiteralPath $CellMedianFile -NoTypeInformation -Encoding utf8 $fixedNames = @('std::vector_reserved', 'std::deque', 'std::list_index_semantics') $bestFixed = @{} $medians | Group-Object profile,type,workload,n | ForEach-Object { $eligible = @($_.Group | Where-Object { $fixedNames -contains $_.structure -or $_.structure -like 'tiered_forced_leaf_*' }) if ($eligible.Count -gt 0) { $key = "$($_.Group[0].profile)|$($_.Group[0].type)|$($_.Group[0].workload)|$($_.Group[0].n)" $bestFixed[$key] = ($eligible.median_ns | Measure-Object -Minimum).Minimum } } $cellScores = @($medians | ForEach-Object { $key = "$($_.profile)|$($_.type)|$($_.workload)|$($_.n)" if ($bestFixed.ContainsKey($key) -and $_.median_ns -gt 0) { [pscustomobject]@{ profile = $_.profile structure = $_.structure type = $_.type workload = $_.workload n = $_.n score = [double]$bestFixed[$key] / [double]$_.median_ns } } }) # Hierarchical aggregation prevents a family with more sizes from receiving a # larger implicit weight: sizes -> workloads -> types -> compiler profiles. $byWorkload = @($cellScores | Group-Object profile,structure,type,workload | ForEach-Object { $g = $_.Group [pscustomobject]@{ profile = $g[0].profile; structure = $g[0].structure type = $g[0].type; workload = $g[0].workload score = Get-GeometricMean @($g.score) } }) $byType = @($byWorkload | Group-Object profile,structure,type | ForEach-Object { $g = $_.Group [pscustomobject]@{ profile = $g[0].profile; structure = $g[0].structure type = $g[0].type; score = Get-GeometricMean @($g.score) } }) $byProfile = @($byType | Group-Object profile,structure | ForEach-Object { $g = $_.Group [pscustomobject]@{ profile = $g[0].profile structure = $g[0].structure score = Get-GeometricMean @($g.score) types = $g.Count cells = @($cellScores | Where-Object { $_.profile -eq $g[0].profile -and $_.structure -eq $g[0].structure }).Count } }) $overall = @($byProfile | Group-Object structure | ForEach-Object { $g = $_.Group [pscustomobject]@{ profile = 'ALL' structure = $g[0].structure score = Get-GeometricMean @($g.score) types = ($g.types | Measure-Object -Maximum).Maximum cells = ($g.cells | Measure-Object -Sum).Sum } }) @($byProfile + $overall) | Sort-Object profile,@{Expression='score';Descending=$true} | Export-Csv -LiteralPath $OutputFile -NoTypeInformation -Encoding utf8 Write-Host "Validated $($rows.Count) raw rows from $($files.Count) profile file(s)." Write-Host "Cell medians: $CellMedianFile" Write-Host "Aggregate: $OutputFile" @($overall | Sort-Object score -Descending) | Format-Table structure,score,types,cells -AutoSize