Cold-Air-Networks/mirror-speedometer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use metrics for CSV (#137)

Browse source 
- Enforce better default order of metrics
- Introduce Metric.separator to be used by metrics-ui and CSV generation
- Use Geomean instead of Total
This commit is contained in:
Camillo Bruni 2023-05-17 01:05:58 +02:00 committed by GitHub
parent daa720b67a
commit 41bdda72e8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 54 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
index.html
View file

@ -73,7 +73,7 @@
<h1 class="section-header">Detailed Results</h1>
<div class="section-content all-metric-results">
<div class="aggregated-metric-result">
<h2>Total</h2>
<h2>Geomean</h2>
<div id="total-chart"></div>
<h2>Tests</h2>
<div id="tests-chart"></div>

57
resources/benchmark-runner.mjs
View file

@ -170,11 +170,18 @@ export class BenchmarkRunner {
this._suites = [WarmupSuite, ...suites];
this._client = client;
this._page = null;
this._metrics = {
__proto__: null,
Total: new Metric("Total"),
Score: new Metric("Score", "score"),
};
this._metrics = null;
this._iterationCount = params.iterationCount;
}
async runMultipleIterations(iterationCount) {
this._iterationCount = iterationCount;
if (this._client?.willStartFirstIteration)
await this._client.willStartFirstIteration(iterationCount);
for (let i = 0; i < iterationCount; i++)
await this._runAllSuites();
if (this._client?.didFinishLastIteration)
await this._client.didFinishLastIteration(this._metrics);
}
_removeFrame() {
@ -212,15 +219,6 @@ export class BenchmarkRunner {
return frame;
}
async runMultipleIterations(iterationCount) {
if (this._client?.willStartFirstIteration)
await this._client.willStartFirstIteration(iterationCount);
for (let i = 0; i < iterationCount; i++)
await this._runAllSuites();
if (this._client?.didFinishLastIteration)
await this._client.didFinishLastIteration(this._metrics);
}
async _runAllSuites() {
this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
@ -352,9 +350,16 @@ export class BenchmarkRunner {
await this._client.didRunSuites(this._measuredValues);
}
}
getIterationTotalMetric(i) {
if (i >= params.iterationCount)
throw new Error(`Requested iteration=${i} does not exist.`);
return this.getMetric(`Iteration-${i}-Total`);
}
_appendIterationMetrics() {
const getMetric = (name) => this._metrics[name] || (this._metrics[name] = new Metric(name));
const getIterationTotalMetric = (i) => getMetric(`Iteration-${i}-Total`);
const collectSubMetrics = (prefix, items, parent) => {
for (let name in items) {
const results = items[name];
@ -366,16 +371,28 @@ export class BenchmarkRunner {
collectSubMetrics(`${metric.name}${Metric.separator}`, results.tests, metric);
}
};
const initializeMetrics = this._metrics === null;
if (initializeMetrics)
this._metrics = { __proto__: null };
const iterationResults = this._measuredValues.tests;
const iterationTotal = getMetric(`Iteration-${this._metrics.Total.length}-Total`);
collectSubMetrics("", iterationResults);
if (initializeMetrics) {
// Prepare all iteration metrics so they are listed at the end of
// of the _metrics object, before "Total" and "Score".
for (let i = 0; i < this._iterationCount; i++)
getIterationTotalMetric(i);
getMetric("Geomean");
getMetric("Score");
}
const iterationTotal = getIterationTotalMetric(this._metrics.Geomean.length);
for (const results of Object.values(iterationResults))
iterationTotal.add(results.total);
iterationTotal.computeAggregatedMetrics();
this._metrics.Total.add(iterationTotal.sum);
this._metrics.Score.add(MILLISECONDS_PER_MINUTE / iterationTotal.sum);
collectSubMetrics("", iterationResults);
getMetric("Geomean").add(iterationTotal.geomean);
getMetric("Score").add(MILLISECONDS_PER_MINUTE / iterationTotal.geomean);
for (const metric of Object.values(this._metrics))
metric.computeAggregatedMetrics();

53
resources/main.mjs
View file

@ -15,6 +15,7 @@ class MainBenchmarkClient {
_progressCompleted = null;
_isRunning = false;
_hasResults = false;
_metrics = Object.create(null);
constructor() {
window.addEventListener("DOMContentLoaded", () => this.prepareUI());
@ -37,6 +38,7 @@ class MainBenchmarkClient {
return false;
}
}
this._metrics = Object.create(null);
this._isRunning = true;
this.developerMode = params.developerMode;
@ -82,6 +84,7 @@ class MainBenchmarkClient {
console.assert(this._isRunning);
this._isRunning = false;
this._hasResults = true;
this._metrics = metrics;
const scoreResults = this._computeResults(this._measuredValuesList, "score");
this._updateGaugeNeedle(scoreResults.mean);
@ -169,7 +172,7 @@ class MainBenchmarkClient {
document.documentElement.style.setProperty("--metrics-line-height", `${trackHeight}px`);
const plotWidth = (params.viewport.width - 120) / 2;
document.getElementById("total-chart").innerHTML = renderMetricView({
metrics: [metrics["Total"]],
metrics: [metrics["Geomean"]],
width: plotWidth,
trackHeight,
renderChildren: false,
@ -201,7 +204,7 @@ class MainBenchmarkClient {
jsonLink.href = URL.createObjectURL(new Blob([jsonData], { type: "application/json" }));
jsonLink.setAttribute("download", `${filePrefix}.json`);
const csvData = this._getFormattedCSVResult();
const csvData = this._formattedCSVResult();
const csvLink = document.getElementById("download-csv");
csvLink.href = URL.createObjectURL(new Blob([csvData], { type: "text/csv" }));
csvLink.setAttribute("download", `${filePrefix}.csv`);
@ -265,49 +268,21 @@ class MainBenchmarkClient {
return JSON.stringify(this._measuredValuesList, undefined, indent);
}
_getFormattedCSVResult() {
let tests = [];
_formattedCSVResult() {
// The CSV format is similar to the details view table. Each measurement is a row with
// the name and N columns with the measurement for each iteration:
// ```
// Measurement,#1,...,#N
// TodoMVC-JavaScript-ES5/Total,num,...,num
// TodoMVC-JavaScript-ES5/Adding100Items,num,...,num
// TodoMVC-JavaScript-ES5/Adding100Items/Sync,num,...,num
// TodoMVC-JavaScript-ES5/Adding100Items/Async,num,...,num
// ...
// TodoMVC-JavaScript-ES6/Total,num,...,num
// TodoMVC-JavaScript-ES6/Adding100Items,num,...,num
// TodoMVC-JavaScript-ES6/Adding100Items/Sync,num,...,num
// TodoMVC-JavaScript-ES6/Adding100Items/Async,num,...,num
// ```
const firstIterationTests = this._measuredValuesList[0].tests;
for (const suiteName in firstIterationTests) {
tests.push([`${suiteName}/Total`]);
for (const testName in firstIterationTests[suiteName].tests) {
tests.push([`${suiteName}/${testName}`]);
for (const subtestName in firstIterationTests[suiteName].tests[testName].tests)
tests.push([`${suiteName}/${testName}/${subtestName}`]);
}
}
// Now push each iteration onto the end of the array
for (const measuredValue of this._measuredValuesList) {
let index = 0;
for (const suiteName in measuredValue.tests) {
const suiteResults = measuredValue.tests[suiteName];
tests[index++].push(suiteResults.total);
for (const testName in suiteResults.tests) {
tests[index++].push(suiteResults.tests[testName].total);
for (const subtestName in suiteResults.tests[testName].tests)
tests[index++].push(suiteResults.tests[testName].tests[subtestName]);
}
}
}
const csv = [["Name"].concat(this._measuredValuesList.map((_, i) => `#${i + 1}`)).join(",")];
for (const test of tests)
csv.push(test.join(","));
const labels = ["Name"];
for (let i = 0; i < params.iterationCount; i++)
labels.push(`#${i + 1}`);
labels.push("Mean");
const metrics = Array.from(Object.values(this._metrics)).filter((metric) => !metric.name.startsWith("Iteration-"));
const metricsValues = metrics.map((metric) => [metric.name, ...metric.values, metric.mean].join(","));
const csv = [labels.join(","), ...metricsValues];
return csv.join("\n");
}
@ -317,7 +292,7 @@ class MainBenchmarkClient {
}
copyCSVResults() {
navigator.clipboard.writeText(this._getFormattedCSVResult());
navigator.clipboard.writeText(this._formattedCSVResult());
}
_showSection(hash) {

5
resources/metric.mjs
View file

@ -22,18 +22,17 @@ export class Metric {
this.values = [];
this.parent = undefined;
this.children = [];
// Mark properties which refer to other Metric objects as
// non-enumerable to avoid issue with JSON.stringify due to circular
// references.
Object.defineProperties(this, {
parent: {
writable: true,
value: undefined,
},
children: {
writable: true,
value: [],
},
});
}