Cold-Air-Networks/mirror-speedometer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cleanup

Browse source 
Merge TodoMVC-React with TodoMVC-React-Window

Reuse TodoMVC-React source and generate TodoMVC-React-Window files in
resources/architecture-examples/react/dist.

Change-Id: I67e37442f594cf17b13b350fc9fa5ec1bf4d6b68

TodoMVC-React windowed variation

Add a variation of the TodoMVC-React workload that renders the app in a
new Window but controls it through scripts loaded in the parent window.

Change-Id: I8b84cf7d9f0e80672808b5bf28959a034e139e0c


Add signpost support

Change-Id: I199b12c21fa25c2daab806988b0b1812a492a3d4


Add suiteForceGC helper

Force run a gc() (expose with `--js-flags=--expose-gc`) before and
after preparing a suite to limit potential ripple-effects from
previous tests.

Change-Id: Ia397b9e0aa8f5dabfb1dc4a69cfad9a7cf63ce60


Add debug features

- Subdomain running options:
  - hosted: using `-$ITER` subdomain suffix
  - local: using increasing port numbers
- Add measure prepare phase for "loading" style metric
- Move the warmup suite to the main config
- Display all top level metrics directly in the details view

The subdomain runner works by the outer default runner communicating
with the inner (embedded) runner via postMessage. There are 2 new
runners

- SubdomainBenchmarkRunner: the outer runner
- EmbeddedBenchmarkRunner: the inner runner

The main page switches over to the SubdomainBenchmarkRunner if the
corresponding param is set.
The EmbeddedBenchmarkRunner is then used by setting `?embedded` and
we end up with double wrapped workload (easy right...):

  SubdomainBenchmarkRunner:
    iframe:
      EmbeddedBenchmarkRunner:
        iframe:
           Suite

This approach doesn't require changing the current workloads and we
can still easily measure framework setup time. Hopefully this can
be cleaned up in the future by refactoring the runner.

Change-Id: Ib813e8d83fd164fb2849b6889f187c1904b21965


Add debug features

- Subdomain running options:
  - hosted: using `-$ITER` subdomain suffix
  - local: using increasing port numbers
- Add measure prepare phase for "loading" style metric
- Move the warmup suite to the main config
- Display all top level metrics directly in the details view

The subdomain runner works by the outer default runner communicating
with the inner (embedded) runner via postMessage. There are 2 new
runners

- SubdomainBenchmarkRunner: the outer runner
- EmbeddedBenchmarkRunner: the inner runner

The main page switches over to the SubdomainBenchmarkRunner if the
corresponding param is set.
The EmbeddedBenchmarkRunner is then used by setting `?embedded` and
we end up with double wrapped workload (easy right...):

  SubdomainBenchmarkRunner:
    iframe:
      EmbeddedBenchmarkRunner:
        iframe:
           Suite

This approach doesn't require changing the current workloads and we
can still easily measure framework setup time. Hopefully this can
be cleaned up in the future by refactoring the runner.

Change-Id: Ib813e8d83fd164fb2849b6889f187c1904b21965
This commit is contained in:
Camillo Bruni 2025-04-24 20:33:04 +02:00
parent 115ae74716
commit 91ede66cc1
22 changed files with 697 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.eslintignore
View file

@ -23,6 +23,8 @@ package-lock.json
!/resources/todomvc/architecture-examples/vue
!/resources/todomvc/architecture-examples/vue-complex
/resources/todomvc/architecture-examples/react/public/window/*
/resources/todomvc/vanilla-examples/*
!/resources/todomvc/vanilla-examples/javascript-es5
!/resources/todomvc/vanilla-examples/javascript-es5-complex

2
.prettierignore
View file

@ -23,6 +23,8 @@ package-lock.json
!/resources/todomvc/architecture-examples/vue
!/resources/todomvc/architecture-examples/vue-complex
/resources/todomvc/architecture-examples/react/public/window/*
/resources/todomvc/vanilla-examples/*
!/resources/todomvc/vanilla-examples/javascript-es5
!/resources/todomvc/vanilla-examples/javascript-es5-complex

2
index.html
View file

@ -86,6 +86,8 @@
<h2>Aggregate Metric</h2>
<div id="geomean-chart"></div>
<h2>Test Metrics Overview</h2>
<div id="toplevel-chart"></div>
<h2>Tests</h2>
<div id="tests-chart"></div>
</div>
<br />

440
resources/benchmark-runner.mjs
View file

@ -10,6 +10,54 @@ export class BenchmarkTestStep {
}
}
// The WarmupSuite is used to make sure all runner helper functions and
// classes are compiled, to avoid unnecessary pauses due to delayed
// compilation of runner methods in the middle of the measuring cycle.
export const WarmupSuite = {
name: "Warmup",
url: "warmup/index.html",
disabled: false,
async prepare(page) {
await page.waitForElement("#testItem");
},
tests: [
// Make sure to run ever page.method once at least
new BenchmarkTestStep("WarmingUpPageMethods", (page) => {
let results = [];
results.push(page.querySelector(".testItem"));
results.push(page.querySelectorAll(".item"));
results.push(page.getElementById("testItem"));
}),
new BenchmarkTestStep("WarmingUpPageElementMethods", (page) => {
const item = page.getElementById("testItem");
item.setValue("value");
item.click();
item.focus();
item.dispatchEvent("change");
item.enter("keypress");
item.dispatchEvent("input");
item.enter("keyup");
}),
new BenchmarkTestStep("WarmingUpPageElementMouseMethods", (page) => {
const item = page.getElementById("testItem");
const mouseEventOptions = { clientX: 100, clientY: 100, bubbles: true, cancelable: true };
const wheelEventOptions = {
clientX: 200,
clientY: 200,
deltaMode: 0,
delta: -10,
deltaY: -10,
bubbles: true,
cancelable: true,
};
item.dispatchEvent("mousedown", mouseEventOptions, MouseEvent);
item.dispatchEvent("mousemove", mouseEventOptions, MouseEvent);
item.dispatchEvent("mouseup", mouseEventOptions, MouseEvent);
item.dispatchEvent("wheel", wheelEventOptions, WheelEvent);
}),
],
};
function getParent(lookupStartNode, path) {
const parent = path.reduce((root, selector) => {
const node = root.querySelector(selector);
@ -22,6 +70,9 @@ function getParent(lookupStartNode, path) {
class Page {
constructor(frame) {
this._frame = frame;
// Downstream-only. this._lookupStartNode is used by the React-Window workload to run the
// querySelector* methods on a child window.
this._lookupStartNode = null;
}
layout() {
@ -58,7 +109,7 @@ class Page {
* @returns PageElement | null
*/
querySelector(selector, path = []) {
const lookupStartNode = this._frame.contentDocument;
const lookupStartNode = this._lookupStartNode ?? this._frame.contentDocument;
const element = getParent(lookupStartNode, path).querySelector(selector);
if (element === null)
@ -82,7 +133,7 @@ class Page {
* @returns array
*/
querySelectorAll(selector, path = []) {
const lookupStartNode = this._frame.contentDocument;
const lookupStartNode = this._lookupStartNode ?? this._frame.contentDocument;
const elements = Array.from(getParent(lookupStartNode, path).querySelectorAll(selector));
for (let i = 0; i < elements.length; i++)
elements[i] = this._wrapElement(elements[i]);
@ -114,6 +165,24 @@ class Page {
_wrapElement(element) {
return new PageElement(element);
}
// Downstream only.
async targetChildWindow() {
return new Promise((resolve) => {
const resolveIfReady = () => {
// Expects the workload to set a global childWindow object.
const maybeChildWindow = this._frame.contentWindow.globalThis.childWindow;
let callback = resolveIfReady;
if (maybeChildWindow) {
this._lookupStartNode = maybeChildWindow.document;
resolve();
return;
}
window.requestAnimationFrame(callback);
};
resolveIfReady();
});
}
}
const NATIVE_OPTIONS = {
@ -215,53 +284,6 @@ function geomeanToScore(geomean) {
return 1000 / geomean;
}
// The WarmupSuite is used to make sure all runner helper functions and
// classes are compiled, to avoid unnecessary pauses due to delayed
// compilation of runner methods in the middle of the measuring cycle.
const WarmupSuite = {
name: "Warmup",
url: "warmup/index.html",
async prepare(page) {
await page.waitForElement("#testItem");
},
tests: [
// Make sure to run ever page.method once at least
new BenchmarkTestStep("WarmingUpPageMethods", (page) => {
let results = [];
results.push(page.querySelector(".testItem"));
results.push(page.querySelectorAll(".item"));
results.push(page.getElementById("testItem"));
}),
new BenchmarkTestStep("WarmingUpPageElementMethods", (page) => {
const item = page.getElementById("testItem");
item.setValue("value");
item.click();
item.focus();
item.dispatchEvent("change");
item.enter("keypress");
item.dispatchEvent("input");
item.enter("keyup");
}),
new BenchmarkTestStep("WarmingUpPageElementMouseMethods", (page) => {
const item = page.getElementById("testItem");
const mouseEventOptions = { clientX: 100, clientY: 100, bubbles: true, cancelable: true };
const wheelEventOptions = {
clientX: 200,
clientY: 200,
deltaMode: 0,
delta: -10,
deltaY: -10,
bubbles: true,
cancelable: true,
};
item.dispatchEvent("mousedown", mouseEventOptions, MouseEvent);
item.dispatchEvent("mousemove", mouseEventOptions, MouseEvent);
item.dispatchEvent("mouseup", mouseEventOptions, MouseEvent);
item.dispatchEvent("wheel", wheelEventOptions, WheelEvent);
}),
],
};
class TestInvoker {
constructor(syncCallback, asyncCallback, reportCallback) {
this._syncCallback = syncCallback;
@ -355,15 +377,48 @@ export class BenchmarkRunner {
this._page = null;
this._metrics = null;
this._iterationCount = params.iterationCount;
this._currentIteration = 0;
if (params.shuffleSeed !== "off")
this._suiteOrderRandomNumberGenerator = seededHashRandomNumberGenerator(params.shuffleSeed);
this._wakeLock = new WakeLock();
this._maybeForceGC = this._initForceGC();
this._useSignPost = !!globalThis.__signpostStart;
}
markStart(groupLabel, startLabel) {
performance.mark(startLabel);
if (this._useSignPost)
globalThis.__signpostStart(groupLabel);
}
markEnd(groupLabel, startLabel, endLabel) {
performance.mark(endLabel);
if (this._useSignPost)
globalThis.__signpostStop(groupLabel);
performance.measure(groupLabel, startLabel, endLabel);
}
_initForceGC() {
if (!params.suiteForceGC)
return () => {};
try {
let gc_extension = eval("gc");
return () => gc_extension({ type: "major", execution: "async" });
} catch (e) {
console.error("suiteForceGC was enabled but --expose-gc was not set", e);
}
return () => {};
}
get currentIteration() {
return this._currentIteration;
}
async runMultipleIterations(iterationCount) {
this._iterationCount = iterationCount;
this._currentIteration = 0;
if (this._client?.willStartFirstIteration)
await this._client.willStartFirstIteration(iterationCount);
await this._client.willStartFirstIteration( this._iterationCount);
try {
await this._runMultipleIterations();
@ -382,25 +437,29 @@ export class BenchmarkRunner {
async _runMultipleIterations() {
const iterationStartLabel = "iteration-start";
const iterationEndLabel = "iteration-end";
for (let i = 0; i < this._iterationCount; i++) {
performance.mark(iterationStartLabel);
for (; this._currentIteration < this._iterationCount; this._currentIteration++) {
const iterationLabel = `iteration-${this._currentIteration}`;
this.markStart(iterationLabel, iterationStartLabel);
await this._runAllSuites();
performance.mark(iterationEndLabel);
performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel);
this.markEnd(iterationLabel, iterationStartLabel, iterationEndLabel);
}
}
_removeFrame() {
if (this._frame) {
if (params.leakyIframes)
if (params.leakyIframes) {
document.getElementById("leakedIframes").append(this._frame);
else
} else {
// Downstream only. The React-Window workload sets a global childWindow object.
if (this._frame.contentWindow.globalThis.childWindow)
this._frame.contentWindow.globalThis.childWindow.close();
this._frame.parentNode.removeChild(this._frame);
}
this._frame = null;
}
}
async _appendFrame(src) {
async _appendFrame() {
const frame = document.createElement("iframe");
const style = frame.style;
style.width = `${params.viewport.width}px`;
@ -409,9 +468,14 @@ export class BenchmarkRunner {
style.position = "absolute";
frame.setAttribute("scrolling", "no");
frame.className = "test-runner";
style.left = "50%";
style.top = "50%";
style.transform = "translate(-50%, -50%)";
if (params.embedded) {
style.left = "0px";
style.top = "0px";
} else {
style.left = "50%";
style.top = "50%";
style.transform = "translate(-50%, -50%)";
}
if (this._client?.willAddTestFrame)
await this._client.willAddTestFrame(frame);
@ -425,10 +489,11 @@ export class BenchmarkRunner {
this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
await this._wakeLock.request();
const prepareLabel = "runner-prepare";
const prepareStartLabel = "runner-prepare-start";
const prepareEndLabel = "runner-prepare-end";
performance.mark(prepareStartLabel);
this.markStart(prepareLabel, prepareStartLabel);
this._removeFrame();
await this._appendFrame();
this._page = new Page(this._frame);
@ -444,8 +509,7 @@ export class BenchmarkRunner {
suites[j] = tmp;
}
}
performance.mark(prepareEndLabel);
performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel);
this.markEnd(prepareLabel, prepareStartLabel, prepareEndLabel);
try {
for (const suite of suites) {
@ -459,46 +523,53 @@ export class BenchmarkRunner {
}
async _finishRunAllSuites() {
const finalizeLabel = "runner-finalize";
const finalizeStartLabel = "runner-finalize-start";
const finalizeEndLabel = "runner-finalize-end";
performance.mark(finalizeStartLabel);
this.markStart(finalizeLabel, finalizeStartLabel);
// Remove frame to clear the view for displaying the results.
this._removeFrame();
await this._finalize();
performance.mark(finalizeEndLabel);
performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel);
this.markEnd(finalizeLabel, finalizeStartLabel, finalizeEndLabel);
await this._wakeLock.release();
}
async _runSuite(suite) {
const suiteName = suite.name;
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
const suiteStartLabel = `suite-${suiteName}-start`;
const suiteEndLabel = `suite-${suiteName}-end`;
this._prepareMetrics(suite);
const suitePrepareLabel = `suite-${suite.name}-prepare`;
const suitePrepareStartLabel = `suite-${suite.name}-prepare-start`;
const suitePrepareEndLabel = `suite-${suite.name}-prepare-end`;
const suiteLabel = `suite-${suite.name}`;
const suiteStartLabel = `suite-${suite.name}-start`;
const suiteEndLabel = `suite-${suite.name}-end`;
performance.mark(suitePrepareStartLabel);
await this._maybeForceGC();
this.markStart(suitePrepareLabel, suitePrepareStartLabel);
const prepareStartTime = performance.now();
await this._prepareSuite(suite);
performance.mark(suitePrepareEndLabel);
const prepareEndTime = performance.now();
this.markEnd(suitePrepareLabel, suitePrepareStartLabel, suitePrepareEndLabel);
performance.mark(suiteStartLabel);
await this._maybeForceGC();
if (params.measurePrepare) {
const prepareTime = prepareEndTime - prepareStartTime;
this._recordPrepareMetric(suite, prepareTime);
}
this.markStart(suiteLabel, suiteStartLabel);
for (const test of suite.tests)
await this._runTestAndRecordResults(suite, test);
performance.mark(suiteEndLabel);
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
this._validateSuiteTotal(suiteName);
this.markEnd(suiteLabel, suiteStartLabel, suiteEndLabel);
}
_validateSuiteTotal(suiteName) {
// When the test is fast and the precision is low (for example with Firefox'
// privacy.resistFingerprinting preference), it's possible that the measured
// total duration for an entire is 0.
const suiteTotal = this._measuredValues.tests[suiteName].total;
if (suiteTotal === 0)
throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`);
_prepareMetrics(suite) {
this._measuredValues.tests[suite.name] = {
tests: {},
total: 0,
};
}
async _prepareSuite(suite) {
@ -512,14 +583,22 @@ export class BenchmarkRunner {
});
}
_recordPrepareMetric(suite, prepareTime) {
const suiteResults = this._measuredValues.tests[suite.name];
suiteResults.tests.Prepare = prepareTime;
suiteResults.total += prepareTime;
}
async _runTestAndRecordResults(suite, test) {
/* eslint-disable-next-line no-async-promise-executor */
if (this._client?.willRunTest)
await this._client.willRunTest(suite, test);
// Prepare all mark labels outside the measuring loop.
const startLabel = `${suite.name}.${test.name}-start`;
const syncLabel = `${suite.name}.${test.name}`;
const syncStartLabel = `${suite.name}.${test.name}-start`;
const syncEndLabel = `${suite.name}.${test.name}-sync-end`;
const asyncLabel = `${suite.name}.${test.name}-async`;
const asyncStartLabel = `${suite.name}.${test.name}-async-start`;
const asyncEndLabel = `${suite.name}.${test.name}-async-end`;
@ -528,22 +607,22 @@ export class BenchmarkRunner {
let asyncTime;
const runSync = () => {
if (params.warmupBeforeSync) {
performance.mark("warmup-start");
this.markStart("warmup", "warmup-start");
const startTime = performance.now();
// Infinite loop for the specified ms.
while (performance.now() - startTime < params.warmupBeforeSync)
continue;
performance.mark("warmup-end");
this.markEnd("warmup", "warmup-start", "warmup-end");
}
performance.mark(startLabel);
this.markStart(syncLabel, syncStartLabel);
const syncStartTime = performance.now();
test.run(this._page);
const syncEndTime = performance.now();
performance.mark(syncEndLabel);
this.markEnd(syncLabel, syncStartLabel, syncEndLabel);
syncTime = syncEndTime - syncStartTime;
performance.mark(asyncStartLabel);
this.markStart(asyncLabel, asyncStartLabel);
asyncStartTime = performance.now();
};
const measureAsync = () => {
@ -553,11 +632,7 @@ export class BenchmarkRunner {
const asyncEndTime = performance.now();
asyncTime = asyncEndTime - asyncStartTime;
this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
performance.mark(asyncEndLabel);
if (params.warmupBeforeSync)
performance.measure("warmup", "warmup-start", "warmup-end");
performance.measure(`${suite.name}.${test.name}-sync`, startLabel, syncEndLabel);
performance.measure(`${suite.name}.${test.name}-async`, asyncStartLabel, asyncEndLabel);
this.markEnd(asyncLabel, asyncStartLabel, asyncEndLabel);
};
const report = () => this._recordTestResults(suite, test, syncTime, asyncTime);
const invokerClass = params.measurementMethod === "raf" ? RAFTestInvoker : TimerTestInvoker;
@ -568,13 +643,18 @@ export class BenchmarkRunner {
async _recordTestResults(suite, test, syncTime, asyncTime) {
// Skip reporting updates for the warmup suite.
if (suite === WarmupSuite)
if (suite.name === "Warmup")
return;
const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 };
const suiteResults = this._measuredValues.tests[suite.name];
const total = syncTime + asyncTime;
this._measuredValues.tests[suite.name] = suiteResults;
suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
suiteResults.tests[test.name] = {
tests: {
Sync: syncTime,
Async: asyncTime,
},
total: total
};
suiteResults.total += total;
if (this._client?.didRunTest)
@ -611,16 +691,24 @@ export class BenchmarkRunner {
throw new Error(`Requested iteration=${i} does not exist.`);
return getMetric(`Iteration-${i}-Total`);
};
const aggregates = Object.create(null);
const collectSubMetrics = (prefix, items, parent) => {
for (let name in items) {
const results = items[name];
const metric = getMetric(prefix + name);
metric.add(results.total ?? results);
const value = results.total ?? results;
metric.add(value);
if (metric.parent !== parent)
parent.addChild(metric);
if (results.tests)
if (results.tests) {
collectSubMetrics(`${metric.name}${Metric.separator}`, results.tests, metric);
} else {
let aggregate = aggregates[name];
if (!aggregate)
aggregate = aggregates[name] = new Metric(name);
aggregate.add(value);
}
}
};
const initializeMetrics = this._metrics === null;
@ -643,6 +731,10 @@ export class BenchmarkRunner {
const iterationTotal = iterationTotalMetric(geomean.length);
for (const results of Object.values(iterationResults))
iterationTotal.add(results.total);
for (let aggregate of Object.values(aggregates)) {
aggregate.computeAggregatedMetrics();
getMetric(aggregate.name).add(aggregate.geomean);
}
iterationTotal.computeAggregatedMetrics();
geomean.add(iterationTotal.geomean);
getMetric("Score").add(geomeanToScore(iterationTotal.geomean));
@ -651,3 +743,145 @@ export class BenchmarkRunner {
metric.computeAggregatedMetrics();
}
}
class PingBackDone {}
const DONE = new PingBackDone();
class PostMessageRunner extends BenchmarkRunner {
_postMessageHandlers = { __proto__: null };
constructor(suites, client) {
super(suites, client);
globalThis.addEventListener("message", this.onMessage.bind(this));
}
onMessage(event) {
const { cmd, data = undefined } = event.data;
const resolver = this._postMessageHandlers[cmd];
if (!resolver)
throw Error(`Unknown onMessage: ${cmd}`);
if (resolver === DONE)
throw Error(`Cannot evaluate ping back resolver twice: ${cmd}`);
this._postMessageHandlers[cmd] = DONE;
resolver(data);
}
async pingBackMessage(name) {
if (this._postMessageHandlers[name] !== DONE)
throw Error(`Unknown ping back message: ${name}`);
return new Promise(resolver => {
this._postMessageHandlers[name] = resolver;
});
}
}
export class SubdomainBenchmarkRunner extends PostMessageRunner {
_finishPromise;
_postMessageHandlers = {
__proto__: null,
prepareDone: DONE,
recordPrepareMetric: DONE,
recordTestResults: DONE,
};
// Assume all workloads report the metrics back via postMessage.
async _runSuite(suite) {
this._prepareMetrics(suite);
await this._prepareSuite(suite);
if (params.measurePrepare) {
const { prepareTime } = await this.pingBackMessage("recordPrepareMetric");
this._recordPrepareMetric(suite, prepareTime);
}
for (const test of suite.tests)
await this._runTestAndRecordResults(suite, test);
}
async _runTestAndRecordResults(suite, test) {
this.postMessage("runTestAndRecordResults");
const { syncTime, asyncTime } = await this.pingBackMessage("recordTestResults");
await this._recordTestResults(suite, test, syncTime, asyncTime);
}
postMessage(cmd, data) {
this._frame.contentWindow.postMessage({ cmd, data }, "*");
}
async _prepareSuite(suite) {
await new Promise((resolve) => {
const frame = this._page._frame;
frame.onload = async () => resolve();
const url = new URL(window.location.href);
const customParams = params.copy();
customParams.embedded = true;
customParams.suites = [suite.name];
customParams.tags = [];
customParams.startAutomatically = true;
customParams.developerMode = false;
customParams.iterationCount = 1;
// TODO: handle local test URLs better
if (this._isLocalUrl(url)) {
url.port = parseInt(url.port) + this._currentIteration;
} else {
// Input: foo-bar.custom.domain.com, foo-bar-0.custom.domain.com
// Output: foo-bar-1.custom.domain.com, foo-bar-1.custom.domain.com
const hostParts = url.hostname.split(".");
const domainWithoutIndex = hostParts[0].match(DOMAIN_WITHOUT_INDEX_RE).groups.nonIndexed;
hostParts[0] = `${domainWithoutIndex}-${this.currentIteration}`;
url.host = hostParts.join(".");
}
url.search = customParams.toSearchParams();
frame.src = url.toString();
});
await this.pingBackMessage("prepareDone");
this.postMessage("startSuite");
}
_isLocalUrl(url) {
return url.hostname === "127.0.0.1" || url.hostname === "localhost";
}
}
const DOMAIN_WITHOUT_INDEX_RE = /(?<nonIndexed>([^-]+(-[^0-9])?)+)(-[0-9]+)?/;
export class EmbeddedBenchmarkRunner extends PostMessageRunner {
_postMessageHandlers = {
__proto__: null,
startSuite: DONE,
runTestAndRecordResults: DONE,
};
constructor(suites, client) {
super(suites, client);
if (globalThis === globalThis.parent) {
const msg = "Cannot use embedded runner in toplevel iframe";
alert(msg);
throw Error(msg);
}
}
postMessage(cmd, data) {
globalThis.parent.postMessage({ cmd, data }, "*");
}
async _prepareSuite(suite) {
await super._prepareSuite(suite);
this.postMessage("prepareDone", { prepareTime: 0 });
await this.pingBackMessage("startSuite");
}
async _runTestAndRecordResults(suite, test) {
await this.pingBackMessage("runTestAndRecordResults");
return super._runTestAndRecordResults(suite, test);
}
_recordPrepareMetric(suite, prepareTime) {
super._recordPrepareMetric(suite, prepareTime);
this.postMessage("recordPrepareMetric", { prepareTime });
}
async _recordTestResults(suite, test, syncTime, asyncTime) {
this.postMessage("recordTestResults", { syncTime, asyncTime });
}
}

62
resources/developer-mode.mjs
View file

@ -23,8 +23,10 @@ export function createDeveloperModeContainer() {
settings.append(createUIForWarmupBeforeSync());
settings.append(createUIForSyncStepDelay());
settings.append(createUIForLeakyIframes());
settings.append(createUIForMeasurePrepare());
settings.append(createUIForDomainPerIteration());
settings.append(createUIForForceGC());
content.append(document.createElement("hr"));
content.append(settings);
content.append(document.createElement("hr"));
@ -92,6 +94,57 @@ function createUIForWarmupSuite() {
return label;
}
function createUIForMeasurePrepare() {
let check = document.createElement("input");
check.type = "checkbox";
check.id = "measure-prepare";
check.checked = params.measurePrepare;
check.onchange = () => {
params.measurePrepare = check.checked;
updateURL();
};
let label = document.createElement("label");
label.append(check, " ", "Measure Prepare");
return label;
}
function createUIForDomainPerIteration() {
let check = document.createElement("input");
check.type = "checkbox";
check.id = "domain-per-iteration";
check.checked = params.domainPerIteration;
check.onchange = () => {
params.domainPerIteration = check.checked;
updateURL();
};
let label = document.createElement("label");
label.append(check, " ", "Use Subdomain-Runner");
return label;
}
function createUIForForceGC() {
let check = document.createElement("input");
check.type = "checkbox";
check.id = "suite-force-gc";
check.checked = params.suiteForceGC;
check.onchange = () => {
params.suiteForceGC = check.checked;
updateURL();
};
let label = document.createElement("label");
label.append(check, " ", "Force GC before every Suite");
return label;
}
function createUIForIterationCount() {
const { range, label } = createTimeRangeUI("Iterations: ", params.iterationCount, "#", 1, 200);
range.onchange = () => {
@ -291,7 +344,7 @@ function updateURL() {
else
url.searchParams.delete("measurementMethod");
const boolParamKeys = ["iterationCount", "useWarmupSuite", "warmupBeforeSync", "waitBeforeSync", "leakyIframes"];
const boolParamKeys = ["iterationCount", "useWarmupSuite", "warmupBeforeSync", "waitBeforeSync", "domainPerIteration", "suiteForceGC", "leakyIframes"];
for (const paramKey of boolParamKeys) {
if (params[paramKey] !== defaultParams[paramKey])
url.searchParams.set(paramKey, params[paramKey]);
@ -299,6 +352,11 @@ function updateURL() {
url.searchParams.delete(paramKey);
}
if (params.measurePrepare !== defaultParams.measurePrepare)
url.searchParams.set("measurePrepare", params.measurePrepare);
else
url.searchParams.delete("measurePrepare");
// Only push state if changed
url.search = decodeURIComponent(url.search);
if (url.href !== window.location.href)

18
resources/main.css
View file

@ -836,3 +836,21 @@ section#about .note {
color: #fff;
stroke: #fff;
}
.embedded section {
border: none;
position: absolute;
top: 0;
left: 0;
margin: 0;
padding: 0;
}
.embedded #logo,
.embedded #progress,
.embedded #info,
.embedded #info-progress,
.embedded #info-label {
display: none !important;
}

32
resources/main.mjs
View file

@ -1,4 +1,4 @@
import { BenchmarkRunner } from "./benchmark-runner.mjs";
import { BenchmarkRunner, SubdomainBenchmarkRunner, EmbeddedBenchmarkRunner } from "./benchmark-runner.mjs";
import * as Statistics from "./statistics.mjs";
import { Suites } from "./tests.mjs";
import { renderMetricView } from "./metric-ui.mjs";
@ -12,11 +12,13 @@ class MainBenchmarkClient {
suitesCount = null;
_measuredValuesList = [];
_finishedTestCount = 0;
_totalSubtestsCount = 0;
_progressCompleted = null;
_isRunning = false;
_hasResults = false;
_developerModeContainer = null;
_metrics = Object.create(null);
_runner = null;
constructor() {
window.addEventListener("DOMContentLoaded", () => this.prepareUI());
@ -60,14 +62,19 @@ class MainBenchmarkClient {
this._isRunning = true;
const enabledSuites = Suites.filter((suite) => !suite.disabled);
const totalSubtestsCount = enabledSuites.reduce((testsCount, suite) => {
this._totalSubtestsCount = enabledSuites.reduce((testsCount, suite) => {
return testsCount + suite.tests.length;
}, 0);
this.stepCount = params.iterationCount * totalSubtestsCount;
this.stepCount = params.iterationCount * this._totalSubtestsCount;
this._progressCompleted.max = this.stepCount;
this.suitesCount = enabledSuites.length;
const runner = new BenchmarkRunner(Suites, this);
runner.runMultipleIterations(params.iterationCount);
if (params.embedded)
this._runner = new EmbeddedBenchmarkRunner(Suites, this);
else if (params.domainPerIteration)
this._runner = new SubdomainBenchmarkRunner(Suites, this);
else
this._runner = new BenchmarkRunner(Suites, this);
this._runner.runMultipleIterations(params.iterationCount);
return true;
}
@ -76,9 +83,6 @@ class MainBenchmarkClient {
}
willAddTestFrame(frame) {
frame.style.left = "50%";
frame.style.top = "50%";
frame.style.transform = "translate(-50%, -50%)";
}
willRunTest(suite, test) {
@ -97,7 +101,7 @@ class MainBenchmarkClient {
willStartFirstIteration() {
this._measuredValuesList = [];
this._finishedTestCount = 0;
this._finishedTestCount = params.currentIteration * this._totalSubtestsCount;
}
didFinishLastIteration(metrics) {
@ -105,6 +109,9 @@ class MainBenchmarkClient {
this._isRunning = false;
this._hasResults = true;
this._metrics = metrics;
// Don't display the results if the runnier is embedded.
if (params.embedded)
return;
const scoreResults = this._computeResults(this._measuredValuesList, "score");
if (scoreResults.isValid)
@ -213,8 +220,9 @@ class MainBenchmarkClient {
const trackHeight = 24;
document.documentElement.style.setProperty("--metrics-line-height", `${trackHeight}px`);
const plotWidth = (params.viewport.width - 120) / 2;
document.getElementById("geomean-chart").innerHTML = renderMetricView({
metrics: [metrics.Geomean],
const singleToplevelMetrics = Object.values(metrics).filter((each) => each && each.unit === "ms" && !each.parent && !each.children.length);
document.getElementById("toplevel-chart").innerHTML = renderMetricView({
metrics: singleToplevelMetrics,
width: plotWidth,
trackHeight,
renderChildren: false,
@ -258,6 +266,8 @@ class MainBenchmarkClient {
}
prepareUI() {
if (params.embedded)
document.body.className = "embedded";
window.addEventListener("hashchange", this._hashChangeHandler.bind(this));
window.addEventListener("resize", this._resizeScreeHandler.bind(this));
this._resizeScreeHandler();

10
resources/metric.mjs
View file

@ -5,7 +5,7 @@ export const MILLISECONDS_PER_MINUTE = 60 * 1000;
export class Metric {
static separator = "/";
constructor(name, unit = "ms") {
constructor(name, unit = "ms", values = []) {
if (typeof name !== "string")
throw new Error(`Invalid metric.name=${name}, expected string.`);
this.name = name;
@ -26,11 +26,17 @@ export class Metric {
this.min = 0.0;
this.max = 0.0;
this.values = [];
this.values = values;
// Mark properties which refer to other Metric objects as
// non-enumerable to avoid issue with JSON.stringify due to circular
// references.
Object.defineProperties(this, {
children: {
writable: true,
value: [],
},
});
Object.defineProperties(this, {
parent: {
writable: true,

71
resources/params.mjs
View file

@ -1,4 +1,4 @@
class Params {
export class Params {
viewport = {
width: 800,
height: 600,
@ -27,11 +27,19 @@ class Params {
shuffleSeed = "off";
// Leak iframes to DevTools can display sources.
leakyIframes = false;
// Emit metrics for the prepare phase.
measurePrepare = false;
// Use a new subdomain for each iteration to avoid caching.
domainPerIteration = false;
// Force a gc before every suite (chrome specific)
suiteForceGC = false;
currentIteration = 0;
embedded = false;
constructor(searchParams = undefined) {
constructor(searchParams = undefined, freeze = true) {
if (searchParams)
this._copyFromSearchParams(searchParams);
if (!this.developerMode) {
if (!this.developerMode && freeze) {
Object.freeze(this.viewport);
Object.freeze(this);
}
@ -65,24 +73,44 @@ class Params {
if (searchParams.has("suite") || searchParams.has("suites")) {
if (searchParams.has("suite") && searchParams.has("suites"))
throw new Error("Params 'suite' and 'suites' can not be used together.");
const value = searchParams.get("suite") || searchParams.get("suites");
this.suites = value.split(",");
if (this.suites.length === 0)
throw new Error("No suites selected");
const value = (searchParams.get("suite") || searchParams.get("suites")).trim();
if (value.length ) {
this.suites = value.split(",");
if (this.suites.length === 0)
throw new Error("No suites selected");
}
searchParams.delete("suite");
searchParams.delete("suites");
}
if (searchParams.has("tags")) {
if (this.suites.length)
throw new Error("'suites' and 'tags' cannot be used together.");
this.tags = searchParams.get("tags").split(",");
const rawTags = searchParams.get("tags");
if (rawTags) {
this.tags = rawTags.split(",");
if (this.tags.length && this.suites.length)
throw new Error("'suites' and 'tags' cannot be used together.");
}
searchParams.delete("tags");
}
this.embedded = searchParams.has("embedded");
searchParams.delete("embedded");
this.developerMode = searchParams.has("developerMode");
searchParams.delete("developerMode");
this.measurePrepare = searchParams.has("measurePrepare");
searchParams.delete("measurePrepare");
this.domainPerIteration = searchParams.has("domainPerIteration");
searchParams.delete("domainPerIteration");
this.suiteForceGC = searchParams.has("suiteForceGC");
searchParams.delete("suiteForceGC");
this.currentIteration = searchParams.get("currentIteration") | 0;
searchParams.delete("currentIteration");
if (searchParams.has("useWarmupSuite")) {
this.useWarmupSuite = true;
searchParams.delete("useWarmupSuite");
@ -134,10 +162,26 @@ class Params {
console.error("Got unused search params", unused);
}
toSearchParamsObject() {
const rawParams = { __proto__: null };
for (const [key, value] of Object.entries(this)) {
if (value === defaultParams[key])
continue;
rawParams[key] = value;
}
// Either suites or params can be used at the same time.
if (rawParams.suites?.length && rawParams.tags?.length)
delete rawParams.suites;
rawParams.viewport = `${this.viewport.width}x${this.viewport.height}`;
return new URLSearchParams(rawParams);
}
toSearchParams() {
const rawParams = { ...this };
rawParams["viewport"] = `${this.viewport.width}x${this.viewport.height}`;
return new URLSearchParams(rawParams).toString();
return this.toSearchParamsObject().toString();
}
copy() {
return new Params(this.toSearchParamsObject(), false);
}
}
@ -152,3 +196,4 @@ try {
alert(`Invalid URL Param: ${e}`);
}
export const params = maybeCustomParams;
globalThis.params = maybeCustomParams;

34
resources/tests.mjs
View file

@ -304,6 +304,39 @@ Suites.push({
],
});
// Downstream only.
Suites.push({
name: "TodoMVC-React-Window",
url: "todomvc/architecture-examples/react/dist/index-window.html#/home",
tags: ["todomvc"],
disabled: true,
async prepare(page) {
await page.targetChildWindow();
const element = await page.waitForElement(".new-todo");
element.focus();
},
tests: [
new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
const newTodo = page.querySelector(".new-todo");
for (let i = 0; i < numberOfItemsToAdd; i++) {
newTodo.setValue(getTodoText(defaultLanguage, i));
newTodo.dispatchEvent("input");
newTodo.enter("keydown");
}
}),
new BenchmarkTestStep("CompletingAllItems", (page) => {
const checkboxes = page.querySelectorAll(".toggle");
for (let i = 0; i < numberOfItemsToAdd; i++)
checkboxes[i].click();
}),
new BenchmarkTestStep("DeletingAllItems", (page) => {
const deleteButtons = page.querySelectorAll(".destroy");
for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
deleteButtons[i].click();
}),
],
});
Suites.push({
name: "TodoMVC-React-Redux",
url: "todomvc/architecture-examples/react-redux/dist/index.html",
@ -1036,6 +1069,7 @@ Suites.push({
],
});
Object.freeze(Suites);
Suites.forEach((suite) => {
if (!suite.tags)

4
resources/todomvc/architecture-examples/react/README.md
View file

@ -6,6 +6,8 @@ This application uses React 17.0.2 to implement a todo application.
- [React](https://reactjs.org/) is a JavaScript library for creating user interfaces.
The downstream version also generates the TodoMVC-React-Window workload. A variation of TodoMVC-React that renders and controls the app in a new window using the scripts loaded in the parent window.
## Implementation details
React:\
@ -22,6 +24,8 @@ Controller: serves as an interface between view & model components
To build the static files, this application utilizes webpack. It minifies and optimizes output files and copies all necessary files to a `dist` folder.
For TodoMVC-React-Window, the webpack scripts separate the JavaScript from the CSS. The JavaScript is loaded by index.html in the main window, while the css is loaded by todo.html in the child window.
## Requirements
The only requirement is an installation of Node, to be able to install dependencies and run scripts to serve a local server.

1
resources/todomvc/architecture-examples/react/dist/index-window.html vendored Normal file
View file

@ -0,0 +1 @@
<!doctype html><html lang="en" data-framework="react" data-version="17.0.2" data-features=""><head><meta charset="UTF-8"/><meta name="description" content="A TodoMVC workload app for Speedometer!"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta http-equiv="X-UA-Compatible" content="ie=edge"/><title>TodoMVC: React-Window</title><script defer="defer" src="windowed.bundle.js"></script></head></html>

1
resources/todomvc/architecture-examples/react/dist/todo.html vendored Normal file
View file

@ -0,0 +1 @@
<!doctype html><html lang="en" data-framework="react" data-version="17.0.2" data-features=""><head><meta charset="UTF-8"/><meta name="description" content="A TodoMVC workload app for Speedometer!"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta http-equiv="X-UA-Compatible" content="ie=edge"/><link href="windowed.css" rel="stylesheet"><title>TodoMVC: React-Window inner window</title></head><body><section class="todoapp" id="root"></section><footer class="info"><p>Click on input field to write your todo.</p><p>At least two characters are needed to be a valid entry.</p><p>Press 'enter' to add the todo.</p><p>Double-click to edit a todo</p></footer></body></html>

3
resources/todomvc/architecture-examples/react/dist/windowed.bundle.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

69
resources/todomvc/architecture-examples/react/dist/windowed.bundle.js.LICENSE.txt vendored Normal file
View file

@ -0,0 +1,69 @@
/*
object-assign
(c) Sindre Sorhus
@license MIT
*/
/*!
Copyright (c) 2018 Jed Watson.
Licensed under the MIT License (MIT), see
http://jedwatson.github.io/classnames
*/
/**
* @remix-run/router v1.3.3
*
* Copyright (c) Remix Software Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE.md file in the root directory of this source tree.
*
* @license MIT
*/
/**
* React Router v6.8.2
*
* Copyright (c) Remix Software Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE.md file in the root directory of this source tree.
*
* @license MIT
*/
/** @license React v0.20.2
* scheduler.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react-dom.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react-jsx-runtime.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

1
resources/todomvc/architecture-examples/react/dist/windowed.bundle.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

2
resources/todomvc/architecture-examples/react/dist/windowed.css vendored Normal file
View file

File diff suppressed because one or more lines are too long

1
resources/todomvc/architecture-examples/react/dist/windowed.css.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

11
resources/todomvc/architecture-examples/react/public/window/index.html Normal file
View file

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en" data-framework="react" data-version="17.0.2" data-features="">
<head>
<meta charset="UTF-8" />
<meta name="description" content="A TodoMVC workload app for Speedometer!" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
<title>TodoMVC: React-Window</title>
<script defer="defer" src=<%= htmlWebpackPlugin.options.script %> ></script>
</head>
</html>

20
resources/todomvc/architecture-examples/react/public/window/todo.html Normal file
View file

@ -0,0 +1,20 @@
<!DOCTYPE html>
<html lang="en" data-framework="react" data-version="17.0.2" data-features="">
<head>
<meta charset="UTF-8" />
<meta name="description" content="A TodoMVC workload app for Speedometer!" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
<link href=<%= htmlWebpackPlugin.options.styles %> rel="stylesheet">
<title>TodoMVC: React-Window inner window</title>
</head>
<body>
<section class="todoapp" id="root"></section>
<footer class="info">
<p>Click on input field to write your todo.</p>
<p>At least two characters are needed to be a valid entry.</p>
<p>Press 'enter' to add the todo.</p>
<p>Double-click to edit a todo</p>
</footer>
</body>
</html>

26
resources/todomvc/architecture-examples/react/src/index-window.js Normal file
View file

@ -0,0 +1,26 @@
import React from "react";
import { render } from "react-dom";
import { HashRouter, Route, Routes } from "react-router-dom";
import { App } from "./todo/app";
import "todomvc-app-css/index.css";
let childWindow = window.open("todo.html", "_blank", "width=800,height=600");
let AsyncLoop = () => {
if (childWindow.document.getElementById("root")) {
render(
<HashRouter>
<Routes>
<Route path="*" element={<App />} />
</Routes>
</HashRouter>,
childWindow.document.getElementById("root")
);
globalThis.childWindow = childWindow;
} else {
setTimeout(AsyncLoop, 10);
}
};
AsyncLoop();

16
resources/todomvc/architecture-examples/react/webpack.common.js
View file

@ -4,12 +4,28 @@ const path = require("path");
module.exports = {
entry: {
app: path.resolve(__dirname, "src", "index.js"),
windowed: path.resolve(__dirname, "src", "index-window.js"),
},
plugins: [
new HtmlWebpackPlugin({
title: "TodoMVC: React",
chunks: ["app"],
template: path.resolve(__dirname, "public", "index.html"),
}),
new HtmlWebpackPlugin({
title: "TodoMVC: React-Window",
inject: false,
template: path.resolve(__dirname, "public", "window", "index.html"),
filename: "index-window.html",
script: "windowed.bundle.js",
}),
new HtmlWebpackPlugin({
title: "TodoMVC: React-Window inner window",
inject: false,
template: path.resolve(__dirname, "public", "window", "todo.html"),
filename: "todo.html",
styles: "windowed.css",
}),
],
output: {
filename: "[name].bundle.js",