Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 256 additions & 0 deletions src/backend/compare/environment-summary.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
import type { Environment, TrialDetails } from '../db/types.js';
import type {
EnvironmentProperty,
EnvironmentRow,
EnvironmentTable
} from '../../shared/view-types.js';
import { asHumanHz, asHumanMem } from '../../shared/data-format.js';

export interface EnvironmentPropertyGroup {
name: string;
properties: EnvironmentProperty[];
}

/**
* The hardware, software, and denoise details of an environment,
* for the baseline and the change.
*/
export interface EnvironmentSummary {
envId: number;
hostname: string;
ostype: string;
cpu: string;

inBase: boolean;
inChange: boolean;

numDifferences: number;
groups: EnvironmentPropertyGroup[];
}

/** The values of a property, for the baseline and the change. */
class PropertyValues {
public readonly base = new Set<string>();
public readonly change = new Set<string>();
}

/** The properties of a group, in the order they were first reported. */
type GroupValues = Map<string, PropertyValues>;

interface EnvironmentValues {
envId: number;
hostname: string;
ostype: string;
cpu: string;

inBase: boolean;
inChange: boolean;

hardware: GroupValues;
software: GroupValues;
denoise: GroupValues;
}

function addValue(
group: GroupValues,
name: string,
value: string,
isBase: boolean
): void {
let values = group.get(name);
if (values === undefined) {
values = new PropertyValues();
group.set(name, values);
}
if (isBase) {
values.base.add(value);
} else {
values.change.add(value);
}
}

function isKnown(value: string | null): value is string {
return value !== null && Number(value) > 0;
}

function addTrial(env: EnvironmentValues, t: TrialDetails, isBase: boolean) {
if (isBase) {
env.inBase = true;
} else {
env.inChange = true;
}

if (isKnown(t.memory)) {
addValue(env.hardware, 'Memory', asHumanMem(Number(t.memory), 2), isBase);
}
if (isKnown(t.clockspeed)) {
const speed = asHumanHz(Number(t.clockspeed), 2);
addValue(env.hardware, 'Clock Speed', speed, isBase);
}

for (const s of t.software) {
addValue(env.software, s.name, s.version, isBase);
}

if (t.denoise) {
for (const [name, value] of Object.entries(t.denoise)) {
const v = value === null ? 'unknown' : String(value);
addValue(env.denoise, name, v, isBase);
}
}
}

function sameValues(a: Set<string>, b: Set<string>): boolean {
return a.size === b.size && [...a].every((v) => b.has(v));
}

function toGroup(
name: string,
group: GroupValues,
compare: boolean
): EnvironmentPropertyGroup {
const properties: EnvironmentProperty[] = [];
for (const [propName, values] of group) {
properties.push({
name: propName,
base: [...values.base],
change: [...values.change],
differs: compare && !sameValues(values.base, values.change)
});
}
return { name, properties };
}

function toSummary(env: EnvironmentValues): EnvironmentSummary {
// an environment used by only one of the commits is not compared
const compare = env.inBase && env.inChange;

const groups = [
toGroup('Hardware', env.hardware, compare),
toGroup('Software', env.software, compare),
toGroup('Denoise', env.denoise, compare)
].filter((g) => g.properties.length > 0);

let numDifferences = 0;
for (const g of groups) {
numDifferences += g.properties.filter((p) => p.differs).length;
}

return {
envId: env.envId,
hostname: env.hostname,
ostype: env.ostype,
cpu: env.cpu,
inBase: env.inBase,
inChange: env.inChange,
numDifferences,
groups
};
}

/**
* Summarize the hardware, software, and denoise details reported by the
* trials of the baseline and the change, for each environment.
*/
export function summarizeEnvironments(
trials: TrialDetails[],
baseCommitId: string,
changeCommitId: string
): EnvironmentSummary[] {
const envs = new Map<number, EnvironmentValues>();

for (const t of trials) {
const isBase = t.commitid === baseCommitId;
if (!isBase && t.commitid !== changeCommitId) {
continue;
}

let env = envs.get(t.envid);
if (env === undefined) {
env = {
envId: t.envid,
hostname: t.hostname,
ostype: t.ostype,
cpu: t.cpu,
inBase: false,
inChange: false,
hardware: new Map(),
software: new Map(),
denoise: new Map()
};
envs.set(t.envid, env);
}
addTrial(env, t, isBase);
}

return [...envs.values()]
.sort(
(a, b) =>
a.hostname.localeCompare(b.hostname) ||
(a.cpu ?? '').localeCompare(b.cpu ?? '')
)
.map(toSummary);
}

/**
* Combine the summaries of all environments into a single table,
* with a row for each detail reported by any of the environments.
*
* An environment gets separate columns for baseline and change only if
* the two differ.
*/
export function toEnvironmentTable(
summaries: EnvironmentSummary[]
): EnvironmentTable {
const columns = summaries.map((s) => ({
envId: s.envId,
hostname: s.hostname,
ostype: s.ostype,
cpu: s.cpu,
inBase: s.inBase,
inChange: s.inChange,
split: s.numDifferences > 0
}));

const groups = new Map<string, Map<string, EnvironmentRow>>();
for (const name of ['Hardware', 'Software', 'Denoise']) {
groups.set(name, new Map());
}

summaries.forEach((s, i) => {
for (const g of s.groups) {
const rows = groups.get(g.name)!;
for (const p of g.properties) {
let row = rows.get(p.name);
if (row === undefined) {
row = { name: p.name, values: summaries.map(() => null) };
rows.set(p.name, row);
}
row.values[i] = { base: p.base, change: p.change, differs: p.differs };
}
}
});

return {
columns,
groups: [...groups]
.map(([name, rows]) => ({ name, rows: [...rows.values()] }))
.filter((g) => g.rows.length > 0)
};
}

export function environmentsOf(trials: TrialDetails[]): Environment[] {
const envs = new Map<number, Environment>();
for (const t of trials) {
if (!envs.has(t.envid)) {
envs.set(t.envid, {
id: t.envid,
hostname: t.hostname,
ostype: t.ostype,
cpu: t.cpu,
note: t.note
});
}
}
return [...envs.values()].sort((a, b) => a.id - b.id);
}
53 changes: 53 additions & 0 deletions src/backend/compare/html/environment-summary.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
<h2 id="environment">Environment</h2>
{% if (it.table.columns.length > 0) {
const numColumns = 1 + it.table.columns.reduce((n, c) => n + (c.split ? 2 : 1), 0);
const base = `<span class="baseline-badge">${it.baselineHash6}</span>`;
const change = `<span class="change-badge">${it.changeHash6}</span>`;
const valuesOf = (values) => values.length > 0 ? values.join(', ') : '—';
%}<div class="table-responsive">
<table class="table table-sm small">
<thead>
<tr>
<th scope="col" rowspan="2"></th>
{% for (const c of it.table.columns) {
%}<th scope="{%= c.split ? 'colgroup' : 'col' %}"{% if (c.split) { %} colspan="2"{% } %} id="environment-{%= c.envId %}">{%= c.hostname %}<br><span class="fw-normal text-body-secondary">{%= c.cpu %} | {%= c.ostype %}</span></th>
{% }
%}</tr>
<tr>
{% for (const c of it.table.columns) {
if (c.split) {
%}<th scope="col">{%- base %}</th>
<th scope="col">{%- change %}</th>
{% } else {
%}<th scope="col">{%- c.inBase ? base : '' %}{% if (c.inBase && c.inChange) { %} {% } %}{%- c.inChange ? change : '' %}</th>
{% }
}
%}</tr>
</thead>
{% for (const g of it.table.groups) {
%}<tbody>
<tr><th scope="rowgroup" colspan="{%= numColumns %}">{%= g.name %}</th></tr>
{% for (const r of g.rows) {
%}<tr>
<th scope="row">{%= r.name %}</th>
{% r.values.forEach((v, i) => {
if (it.table.columns[i].split) {
if (v === null) {
%}<td></td>
<td></td>
{% } else {
%}<td{% if (v.differs) { %} class="table-warning"{% } %}>{%= valuesOf(v.base) %}</td>
<td{% if (v.differs) { %} class="table-warning"{% } %}>{%= valuesOf(v.change) %}</td>
{% }
} else {
%}<td>{%= v === null ? '' : valuesOf(v.base.length > 0 ? v.base : v.change) %}</td>
{% }
});
%}</tr>
{% }
%}</tbody>
{% }
%}</table>
</div>
{% }
%}
6 changes: 5 additions & 1 deletion src/backend/compare/html/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,11 @@ <h3>Changes in Benchmark Set</h3>
{% if (it.hasExeComparison) {
%}{%- include('compare-exes.html', {...it.stats, config: it.config}) %}{%
}
%}
%}{%- include('environment-summary.html', {
table: it.environmentTable,
baselineHash6: it.baselineHash6,
changeHash6: it.changeHash6
}) %}

</main>
</div></div>
Expand Down
3 changes: 2 additions & 1 deletion src/backend/compare/html/navigation.html
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@
{% }
%}</nav>
{% }
%}</nav>
%}<a href="#environment">Environment</a>
</nav>
20 changes: 16 additions & 4 deletions src/backend/compare/prep-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ import {
} from '../../shared/stats.js';
import type {
CriterionData,
Environment,
MeasurementData,
Measurements,
ProcessedResult,
RevisionComparison,
RevisionData
RevisionData,
TrialDetails
} from '../db/types.js';
import {
AllStats,
Expand Down Expand Up @@ -46,6 +46,11 @@ import { siteAesthetics } from '../../shared/aesthetics.js';

import { collateMeasurements } from './db-data.js';
import { HasProfile } from '../db/has-profile.js';
import {
environmentsOf,
summarizeEnvironments,
toEnvironmentTable
} from './environment-summary.js';
import { assert } from '../../backend/logging.js';
import { getBenchmarkArgumentsAsNamePart } from '../../shared/helpers.js';
import type { ValuesPossiblyMissing } from '../../shared/api.js';
Expand Down Expand Up @@ -1076,7 +1081,7 @@ export function calculateDataForOverviewPlot(

export async function prepareCompareView(
results: MeasurementData[],
environments: Environment[],
trials: TrialDetails[],
hasProfiles: HasProfile | null,
reportId: string,
projectSlug: string,
Expand Down Expand Up @@ -1114,7 +1119,14 @@ export async function prepareCompareView(

notInBoth: allStats.acrossVersions.missing.length > 0,

stats: { ...allStats, environments },
stats: { ...allStats, environments: environmentsOf(trials) },
environmentTable: toEnvironmentTable(
summarizeEnvironments(
trials,
revDetails.baseCommitId,
revDetails.changeCommitId
)
),
config: {
reportsUrl: siteConfig.reportsUrl,
overviewPlotWidth: siteAesthetics.overviewPlotWidth
Expand Down
Loading
Loading