Cold-Air-Networks/mirror-speedometer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Runner Cleanup: SuiteRunner & TestRunner classes (#452)

Browse source
This commit is contained in:
Thorsten Kober 2024-11-15 13:42:38 -05:00 committed by GitHub
parent e70778f1a8
commit 67bc21f1f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 204 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
resources/benchmark-runner.mjs
View file

@ -1,6 +1,6 @@
import { Metric } from "./metric.mjs";
import { params } from "./params.mjs";
import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs";
const performance = globalThis.performance;
@ -223,7 +223,7 @@ function geomeanToScore(geomean) {
// The WarmupSuite is used to make sure all runner helper functions and
// classes are compiled, to avoid unnecessary pauses due to delayed
// compilation of runner methods in the middle of the measuring cycle.
const WarmupSuite = {
export const WarmupSuite = {
name: "Warmup",
url: "warmup/index.html",
async prepare(page) {
@ -410,7 +410,7 @@ export class BenchmarkRunner {
// FIXME: Encapsulate more state in the SuiteRunner.
// FIXME: Return and use measured values from SuiteRunner.
const suiteRunnerClass = SUITE_RUNNER_LOOKUP[suite.type ?? "default"];
const suiteRunner = new suiteRunnerClass(this._measuredValues, this._frame, this._page, this._client, suite);
const suiteRunner = new suiteRunnerClass(this._frame, this._page, params, suite, this._client, this._measuredValues);
await suiteRunner.run();
}
@ -484,151 +484,3 @@ export class BenchmarkRunner {
metric.computeAggregatedMetrics();
}
}
// FIXME: Create AsyncSuiteRunner subclass.
// FIXME: Create RemoteSuiteRunner subclass.
export class SuiteRunner {
constructor(measuredValues, frame, page, client, suite) {
// FIXME: Create SuiteRunner-local measuredValues.
this._suiteResults = measuredValues.tests[suite.name];
if (!this._suiteResults) {
this._suiteResults = { tests: {}, total: 0 };
measuredValues.tests[suite.name] = this._suiteResults;
}
this._measuredValues = measuredValues;
this._frame = frame;
this._page = page;
this._client = client;
this._suite = suite;
}
async run() {
await this._prepareSuite();
await this._runSuite();
}
async _prepareSuite() {
const suiteName = this._suite.name;
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
performance.mark(suitePrepareStartLabel);
await this._loadFrame();
await this._suite.prepare(this._page);
performance.mark(suitePrepareEndLabel);
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
}
async _runSuite() {
const suiteName = this._suite.name;
const suiteStartLabel = `suite-${suiteName}-start`;
const suiteEndLabel = `suite-${suiteName}-end`;
performance.mark(suiteStartLabel);
for (const test of this._suite.tests)
await this._runTestAndRecordResults(test);
performance.mark(suiteEndLabel);
performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
this._validateSuiteTotal();
}
_validateSuiteTotal() {
// When the test is fast and the precision is low (for example with Firefox'
// privacy.resistFingerprinting preference), it's possible that the measured
// total duration for an entire is 0.
const suiteTotal = this._suiteResults.total;
if (suiteTotal === 0)
throw new Error(`Got invalid 0-time total for suite ${this._suite.name}: ${suiteTotal}`);
}
async _loadFrame() {
return new Promise((resolve, reject) => {
const frame = this._page._frame;
frame.onload = () => resolve();
frame.onerror = () => reject();
frame.src = this._suite.url;
});
}
async _runTestAndRecordResults(test) {
if (this._client?.willRunTest)
await this._client.willRunTest(this._suite, test);
// Prepare all mark labels outside the measuring loop.
const suiteName = this._suite.name;
const testName = test.name;
const startLabel = `${suiteName}.${testName}-start`;
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
const asyncEndLabel = `${suiteName}.${testName}-async-end`;
let syncTime;
let asyncStartTime;
let asyncTime;
const runSync = () => {
if (params.warmupBeforeSync) {
performance.mark("warmup-start");
const startTime = performance.now();
// Infinite loop for the specified ms.
while (performance.now() - startTime < params.warmupBeforeSync)
continue;
performance.mark("warmup-end");
}
performance.mark(startLabel);
const syncStartTime = performance.now();
test.run(this._page);
const syncEndTime = performance.now();
performance.mark(syncEndLabel);
syncTime = syncEndTime - syncStartTime;
performance.mark(asyncStartLabel);
asyncStartTime = performance.now();
};
const measureAsync = () => {
// Some browsers don't immediately update the layout for paint.
// Force the layout here to ensure we're measuring the layout time.
const height = this._frame.contentDocument.body.getBoundingClientRect().height;
const asyncEndTime = performance.now();
asyncTime = asyncEndTime - asyncStartTime;
this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
performance.mark(asyncEndLabel);
if (params.warmupBeforeSync)
performance.measure("warmup", "warmup-start", "warmup-end");
const suiteName = this._suite.name;
const testName = test.name;
performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel);
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
};
const report = () => this._recordTestResults(test, syncTime, asyncTime);
const invokerClass = TEST_INVOKER_LOOKUP[params.measurementMethod];
const invoker = new invokerClass(runSync, measureAsync, report, params);
return invoker.start();
}
async _recordTestResults(test, syncTime, asyncTime) {
// Skip reporting updates for the warmup suite.
if (this._suite === WarmupSuite)
return;
const total = syncTime + asyncTime;
this._suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
this._suiteResults.total += total;
if (this._client?.didRunTest)
await this._client.didRunTest(this._suite, test);
}
}
// FIXME: implement remote steps
class RemoteSuiteRunner extends SuiteRunner {}
const SUITE_RUNNER_LOOKUP = {
__proto__: null,
default: SuiteRunner,
remote: RemoteSuiteRunner,
};

104
resources/suite-runner.mjs Normal file
View file

@ -0,0 +1,104 @@
import { TestRunner } from "./test-runner.mjs";
import { WarmupSuite } from "./benchmark-runner.mjs";
// FIXME: Create AsyncSuiteRunner subclass.
// FIXME: Create RemoteSuiteRunner subclass.
export class SuiteRunner {
#frame;
#page;
#params;
#suite;
#client;
#suiteResults;
constructor(frame, page, params, suite, client, measuredValues) {
// FIXME: Create SuiteRunner-local measuredValues.
this.#suiteResults = measuredValues.tests[suite.name];
if (!this.#suiteResults) {
this.#suiteResults = { tests: {}, total: 0 };
measuredValues.tests[suite.name] = this.#suiteResults;
}
this.#frame = frame;
this.#page = page;
this.#client = client;
this.#suite = suite;
this.#params = params;
}
async run() {
await this._prepareSuite();
await this._runSuite();
}
async _prepareSuite() {
const suiteName = this.#suite.name;
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
performance.mark(suitePrepareStartLabel);
await this._loadFrame();
await this.#suite.prepare(this.#page);
performance.mark(suitePrepareEndLabel);
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
}
async _runSuite() {
const suiteName = this.#suite.name;
const suiteStartLabel = `suite-${suiteName}-start`;
const suiteEndLabel = `suite-${suiteName}-end`;
performance.mark(suiteStartLabel);
for (const test of this.#suite.tests) {
if (this.#client?.willRunTest)
await this.#client.willRunTest(this.#suite, test);
const testRunner = new TestRunner(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults);
await testRunner.runTest();
}
performance.mark(suiteEndLabel);
performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
this._validateSuiteTotal();
}
_validateSuiteTotal() {
// When the test is fast and the precision is low (for example with Firefox'
// privacy.resistFingerprinting preference), it's possible that the measured
// total duration for an entire is 0.
const suiteTotal = this.#suiteResults.total;
if (suiteTotal === 0)
throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`);
}
async _loadFrame() {
return new Promise((resolve, reject) => {
const frame = this.#frame;
frame.onload = () => resolve();
frame.onerror = () => reject();
frame.src = this.#suite.url;
});
}
_recordTestResults = async (test, syncTime, asyncTime) => {
// Skip reporting updates for the warmup suite.
if (this.#suite === WarmupSuite)
return;
const total = syncTime + asyncTime;
this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
this.#suiteResults.total += total;
if (this.#client?.didRunTest)
await this.#client.didRunTest(this.#suite, test);
};
}
// FIXME: implement remote steps
class RemoteSuiteRunner extends SuiteRunner {}
export const SUITE_RUNNER_LOOKUP = {
__proto__: null,
default: SuiteRunner,
remote: RemoteSuiteRunner,
};

79
resources/test-runner.mjs Normal file
View file

@ -0,0 +1,79 @@
import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
export class TestRunner {
#frame;
#page;
#params;
#suite;
#test;
#callback;
constructor(frame, page, params, suite, test, callback) {
this.#suite = suite;
this.#test = test;
this.#params = params;
this.#callback = callback;
this.#page = page;
this.#frame = frame;
}
async runTest() {
// Prepare all mark labels outside the measuring loop.
const suiteName = this.#suite.name;
const testName = this.#test.name;
const syncStartLabel = `${suiteName}.${testName}-start`;
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
const asyncEndLabel = `${suiteName}.${testName}-async-end`;
let syncTime;
let asyncStartTime;
let asyncTime;
const runSync = () => {
if (this.#params.warmupBeforeSync) {
performance.mark("warmup-start");
const startTime = performance.now();
// Infinite loop for the specified ms.
while (performance.now() - startTime < this.#params.warmupBeforeSync)
continue;
performance.mark("warmup-end");
}
performance.mark(syncStartLabel);
const syncStartTime = performance.now();
this.#test.run(this.#page);
const syncEndTime = performance.now();
performance.mark(syncEndLabel);
syncTime = syncEndTime - syncStartTime;
performance.mark(asyncStartLabel);
asyncStartTime = performance.now();
};
const measureAsync = () => {
const bodyReference = this.#frame ? this.#frame.contentDocument.body : document.body;
const windowReference = this.#frame ? this.#frame.contentWindow : window;
// Some browsers don't immediately update the layout for paint.
// Force the layout here to ensure we're measuring the layout time.
const height = bodyReference.getBoundingClientRect().height;
windowReference._unusedHeightValue = height; // Prevent dead code elimination.
const asyncEndTime = performance.now();
performance.mark(asyncEndLabel);
asyncTime = asyncEndTime - asyncStartTime;
if (this.#params.warmupBeforeSync)
performance.measure("warmup", "warmup-start", "warmup-end");
performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel);
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
};
const report = () => this.#callback(this.#test, syncTime, asyncTime);
const invokerClass = TEST_INVOKER_LOOKUP[this.#params.measurementMethod];
const invoker = new invokerClass(runSync, measureAsync, report, this.#params);
return invoker.start();
}
}

27
tests/benchmark-runner-tests.mjs
View file

@ -1,4 +1,6 @@
import { BenchmarkRunner, SuiteRunner } from "../resources/benchmark-runner.mjs";
import { BenchmarkRunner } from "../resources/benchmark-runner.mjs";
import { SuiteRunner } from "../resources/suite-runner.mjs";
import { TestRunner } from "../resources/test-runner.mjs";
import { defaultParams } from "../resources/params.mjs";
function TEST_FIXTURE(name) {
@ -148,14 +150,14 @@ describe("BenchmarkRunner", () => {
});
describe("runSuite", () => {
let _prepareSuiteSpy, _loadFrameStub, _runTestAndRecordResultsStub, _validateSuiteTotalStub, _suitePrepareSpy, performanceMarkSpy;
let _prepareSuiteSpy, _loadFrameStub, _runTestStub, _validateSuiteTotalStub, _suitePrepareSpy, performanceMarkSpy;
const suite = SUITES_FIXTURE[0];
before(async () => {
_prepareSuiteSpy = spy(SuiteRunner.prototype, "_prepareSuite");
_loadFrameStub = stub(SuiteRunner.prototype, "_loadFrame").callsFake(async () => null);
_runTestAndRecordResultsStub = stub(SuiteRunner.prototype, "_runTestAndRecordResults").callsFake(async () => null);
_runTestStub = stub(TestRunner.prototype, "runTest").callsFake(async () => null);
_validateSuiteTotalStub = stub(SuiteRunner.prototype, "_validateSuiteTotal").callsFake(async () => null);
performanceMarkSpy = spy(window.performance, "mark");
_suitePrepareSpy = spy(suite, "prepare");
@ -170,7 +172,7 @@ describe("BenchmarkRunner", () => {
});
it("should run and record results for every test in suite", async () => {
assert.calledThrice(_runTestAndRecordResultsStub);
assert.calledThrice(_runTestStub);
assert.calledOnce(_validateSuiteTotalStub);
assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-start");
assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-end");
@ -186,13 +188,14 @@ describe("BenchmarkRunner", () => {
let performanceMarkSpy;
const suite = SUITES_FIXTURE[0];
const params = { measurementMethod: "raf" };
before(async () => {
runner._suite = suite;
await runner._appendFrame();
performanceMarkSpy = spy(window.performance, "mark");
const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, suite);
await suiteRunner._runTestAndRecordResults(suite.tests[0]);
const suiteRunner = new SuiteRunner(runner._frame, runner._page, params, suite, runner._client, runner._measuredValues);
await suiteRunner._runSuite();
});
it("should run client pre and post hooks if present", () => {
@ -205,7 +208,11 @@ describe("BenchmarkRunner", () => {
assert.calledWith(performanceMarkSpy, "Suite 1.Test 1-sync-end");
assert.calledWith(performanceMarkSpy, "Suite 1.Test 1-async-start");
assert.calledWith(performanceMarkSpy, "Suite 1.Test 1-async-end");
expect(performanceMarkSpy.callCount).to.equal(4);
// SuiteRunner adds 2 marks.
// Suite used here contains 3 tests.
// Each Testrunner adds 4 marks.
expect(performanceMarkSpy.callCount).to.equal(14);
});
});
@ -218,6 +225,8 @@ describe("BenchmarkRunner", () => {
const asyncStart = 12000;
const asyncEnd = 13000;
const params = { measurementMethod: "raf" };
before(async () => {
stub(runner, "_measuredValues").value({
tests: {},
@ -226,8 +235,8 @@ describe("BenchmarkRunner", () => {
stubPerformanceNowCalls(syncStart, syncEnd, asyncStart, asyncEnd);
// instantiate recorded test results
const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, suite);
await suiteRunner._runTestAndRecordResults(suite.tests[0]);
const suiteRunner = new SuiteRunner(runner._frame, runner._page, params, suite, runner._client, runner._measuredValues);
await suiteRunner._runSuite();
await runner._finalize();
});