From f40db1000aac6b03e6d2cc7936c3330875023990 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Thu, 24 Apr 2025 20:36:59 +0200 Subject: [PATCH] Merge v3.0-custom changes to v3.1-custom - TodoMVC-React windowed variation - Add signpost support - Add suiteForceGC helper - Subdomain running options: - hosted: using `-$ITER` subdomain suffix - local: using increasing port numbers - Add measure prepare phase for "loading" style metric - Move the warmup suite to the main config - Display all top level metrics directly in the details view Change-Id: Ib813e8d83fd164fb2849b6889f187c1904b21965 --- .eslintignore | 2 + .prettierignore | 2 + index.html | 2 + resources/benchmark-runner.mjs | 348 +++++++++++++++--- resources/developer-mode.mjs | 62 +++- resources/main.css | 18 + resources/main.mjs | 32 +- resources/metric.mjs | 10 +- resources/params.mjs | 71 +++- resources/tests.mjs | 34 ++ .../architecture-examples/react/README.md | 4 + .../react/dist/index-window.html | 1 + .../react/dist/todo.html | 1 + .../react/dist/windowed.bundle.js | 3 + .../react/dist/windowed.bundle.js.LICENSE.txt | 69 ++++ .../react/dist/windowed.bundle.js.map | 1 + .../react/dist/windowed.css | 2 + .../react/dist/windowed.css.map | 1 + .../react/public/window/index.html | 11 + .../react/public/window/todo.html | 20 + .../react/src/index-window.js | 26 ++ .../react/webpack.common.js | 16 + 22 files changed, 651 insertions(+), 85 deletions(-) create mode 100644 resources/todomvc/architecture-examples/react/dist/index-window.html create mode 100644 resources/todomvc/architecture-examples/react/dist/todo.html create mode 100644 resources/todomvc/architecture-examples/react/dist/windowed.bundle.js create mode 100644 resources/todomvc/architecture-examples/react/dist/windowed.bundle.js.LICENSE.txt create mode 100644 resources/todomvc/architecture-examples/react/dist/windowed.bundle.js.map create mode 100644 resources/todomvc/architecture-examples/react/dist/windowed.css create mode 100644 resources/todomvc/architecture-examples/react/dist/windowed.css.map create mode 100644 resources/todomvc/architecture-examples/react/public/window/index.html create mode 100644 resources/todomvc/architecture-examples/react/public/window/todo.html create mode 100644 resources/todomvc/architecture-examples/react/src/index-window.js diff --git a/.eslintignore b/.eslintignore index 431c4a7..a31664b 100644 --- a/.eslintignore +++ b/.eslintignore @@ -23,6 +23,8 @@ package-lock.json !/resources/todomvc/architecture-examples/vue !/resources/todomvc/architecture-examples/vue-complex +/resources/todomvc/architecture-examples/react/public/window/* + /resources/todomvc/vanilla-examples/* !/resources/todomvc/vanilla-examples/javascript-es5 !/resources/todomvc/vanilla-examples/javascript-es5-complex diff --git a/.prettierignore b/.prettierignore index 380c47c..7cb6fb5 100644 --- a/.prettierignore +++ b/.prettierignore @@ -23,6 +23,8 @@ package-lock.json !/resources/todomvc/architecture-examples/vue !/resources/todomvc/architecture-examples/vue-complex +/resources/todomvc/architecture-examples/react/public/window/* + /resources/todomvc/vanilla-examples/* !/resources/todomvc/vanilla-examples/javascript-es5 !/resources/todomvc/vanilla-examples/javascript-es5-complex diff --git a/index.html b/index.html index 9f7f77e..10f14c4 100644 --- a/index.html +++ b/index.html @@ -86,6 +86,8 @@

Aggregate Metric

Test Metrics Overview

+
+

Tests


diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index ebc6853..2c5adbf 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -22,6 +22,9 @@ function getParent(lookupStartNode, path) { class Page { constructor(frame) { this._frame = frame; + // Downstream-only. this._lookupStartNode is used by the React-Window workload to run the + // querySelector* methods on a child window. + this._lookupStartNode = null; } layout() { @@ -58,7 +61,7 @@ class Page { * @returns PageElement | null */ querySelector(selector, path = []) { - const lookupStartNode = this._frame.contentDocument; + const lookupStartNode = this._lookupStartNode ?? this._frame.contentDocument; const element = getParent(lookupStartNode, path).querySelector(selector); if (element === null) @@ -82,7 +85,7 @@ class Page { * @returns array */ querySelectorAll(selector, path = []) { - const lookupStartNode = this._frame.contentDocument; + const lookupStartNode = this._lookupStartNode ?? this._frame.contentDocument; const elements = Array.from(getParent(lookupStartNode, path).querySelectorAll(selector)); for (let i = 0; i < elements.length; i++) elements[i] = this._wrapElement(elements[i]); @@ -114,6 +117,24 @@ class Page { _wrapElement(element) { return new PageElement(element); } + + // Downstream only. + async targetChildWindow() { + return new Promise((resolve) => { + const resolveIfReady = () => { + // Expects the workload to set a global childWindow object. + const maybeChildWindow = this._frame.contentWindow.globalThis.childWindow; + let callback = resolveIfReady; + if (maybeChildWindow) { + this._lookupStartNode = maybeChildWindow.document; + resolve(); + return; + } + window.requestAnimationFrame(callback); + }; + resolveIfReady(); + }); + } } const NATIVE_OPTIONS = { @@ -218,9 +239,10 @@ function geomeanToScore(geomean) { // The WarmupSuite is used to make sure all runner helper functions and // classes are compiled, to avoid unnecessary pauses due to delayed // compilation of runner methods in the middle of the measuring cycle. -const WarmupSuite = { +export const WarmupSuite = { name: "Warmup", url: "warmup/index.html", + disabled: false, async prepare(page) { await page.waitForElement("#testItem"); }, @@ -355,15 +377,48 @@ export class BenchmarkRunner { this._page = null; this._metrics = null; this._iterationCount = params.iterationCount; + this._currentIteration = 0; if (params.shuffleSeed !== "off") this._suiteOrderRandomNumberGenerator = seededHashRandomNumberGenerator(params.shuffleSeed); this._wakeLock = new WakeLock(); + this._maybeForceGC = this._initForceGC(); + this._useSignPost = !!globalThis.__signpostStart; + } + + markStart(groupLabel, startLabel) { + performance.mark(startLabel); + if (this._useSignPost) + globalThis.__signpostStart(groupLabel); + } + + markEnd(groupLabel, startLabel, endLabel) { + performance.mark(endLabel); + if (this._useSignPost) + globalThis.__signpostStop(groupLabel); + performance.measure(groupLabel, startLabel, endLabel); + } + + _initForceGC() { + if (!params.suiteForceGC) + return () => {}; + try { + let gc_extension = eval("gc"); + return () => gc_extension({ type: "major", execution: "async" }); + } catch (e) { + console.error("suiteForceGC was enabled but --expose-gc was not set", e); + } + return () => {}; + } + + get currentIteration() { + return this._currentIteration; } async runMultipleIterations(iterationCount) { this._iterationCount = iterationCount; + this._currentIteration = 0; if (this._client?.willStartFirstIteration) - await this._client.willStartFirstIteration(iterationCount); + await this._client.willStartFirstIteration( this._iterationCount); try { await this._runMultipleIterations(); @@ -382,25 +437,29 @@ export class BenchmarkRunner { async _runMultipleIterations() { const iterationStartLabel = "iteration-start"; const iterationEndLabel = "iteration-end"; - for (let i = 0; i < this._iterationCount; i++) { - performance.mark(iterationStartLabel); + for (; this._currentIteration < this._iterationCount; this._currentIteration++) { + const iterationLabel = `iteration-${this._currentIteration}`; + this.markStart(iterationLabel, iterationStartLabel); await this._runAllSuites(); - performance.mark(iterationEndLabel); - performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel); + this.markEnd(iterationLabel, iterationStartLabel, iterationEndLabel); } } _removeFrame() { if (this._frame) { - if (params.leakyIframes) + if (params.leakyIframes) { document.getElementById("leakedIframes").append(this._frame); - else + } else { + // Downstream only. The React-Window workload sets a global childWindow object. + if (this._frame.contentWindow.globalThis.childWindow) + this._frame.contentWindow.globalThis.childWindow.close(); this._frame.parentNode.removeChild(this._frame); + } this._frame = null; } } - async _appendFrame(src) { + async _appendFrame() { const frame = document.createElement("iframe"); const style = frame.style; style.width = `${params.viewport.width}px`; @@ -409,9 +468,14 @@ export class BenchmarkRunner { style.position = "absolute"; frame.setAttribute("scrolling", "no"); frame.className = "test-runner"; - style.left = "50%"; - style.top = "50%"; - style.transform = "translate(-50%, -50%)"; + if (params.embedded) { + style.left = "0px"; + style.top = "0px"; + } else { + style.left = "50%"; + style.top = "50%"; + style.transform = "translate(-50%, -50%)"; + } if (this._client?.willAddTestFrame) await this._client.willAddTestFrame(frame); @@ -425,10 +489,11 @@ export class BenchmarkRunner { this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN }; await this._wakeLock.request(); + const prepareLabel = "runner-prepare"; const prepareStartLabel = "runner-prepare-start"; const prepareEndLabel = "runner-prepare-end"; - performance.mark(prepareStartLabel); + this.markStart(prepareLabel, prepareStartLabel); this._removeFrame(); await this._appendFrame(); this._page = new Page(this._frame); @@ -444,8 +509,7 @@ export class BenchmarkRunner { suites[j] = tmp; } } - performance.mark(prepareEndLabel); - performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel); + this.markEnd(prepareLabel, prepareStartLabel, prepareEndLabel); try { for (const suite of suites) { @@ -459,46 +523,53 @@ export class BenchmarkRunner { } async _finishRunAllSuites() { + const finalizeLabel = "runner-finalize"; const finalizeStartLabel = "runner-finalize-start"; const finalizeEndLabel = "runner-finalize-end"; - performance.mark(finalizeStartLabel); + this.markStart(finalizeLabel, finalizeStartLabel); // Remove frame to clear the view for displaying the results. this._removeFrame(); await this._finalize(); - performance.mark(finalizeEndLabel); - performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel); + this.markEnd(finalizeLabel, finalizeStartLabel, finalizeEndLabel); await this._wakeLock.release(); } async _runSuite(suite) { - const suiteName = suite.name; - const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`; - const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`; - const suiteStartLabel = `suite-${suiteName}-start`; - const suiteEndLabel = `suite-${suiteName}-end`; + this._prepareMetrics(suite); + const suitePrepareLabel = `suite-${suite.name}-prepare`; + const suitePrepareStartLabel = `suite-${suite.name}-prepare-start`; + const suitePrepareEndLabel = `suite-${suite.name}-prepare-end`; + const suiteLabel = `suite-${suite.name}`; + const suiteStartLabel = `suite-${suite.name}-start`; + const suiteEndLabel = `suite-${suite.name}-end`; - performance.mark(suitePrepareStartLabel); + await this._maybeForceGC(); + + this.markStart(suitePrepareLabel, suitePrepareStartLabel); + const prepareStartTime = performance.now(); await this._prepareSuite(suite); - performance.mark(suitePrepareEndLabel); + const prepareEndTime = performance.now(); + this.markEnd(suitePrepareLabel, suitePrepareStartLabel, suitePrepareEndLabel); - performance.mark(suiteStartLabel); + await this._maybeForceGC(); + + if (params.measurePrepare) { + const prepareTime = prepareEndTime - prepareStartTime; + this._recordPrepareMetric(suite, prepareTime); + } + + this.markStart(suiteLabel, suiteStartLabel); for (const test of suite.tests) await this._runTestAndRecordResults(suite, test); - performance.mark(suiteEndLabel); - - performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); - performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel); - this._validateSuiteTotal(suiteName); + this.markEnd(suiteLabel, suiteStartLabel, suiteEndLabel); } - _validateSuiteTotal(suiteName) { - // When the test is fast and the precision is low (for example with Firefox' - // privacy.resistFingerprinting preference), it's possible that the measured - // total duration for an entire is 0. - const suiteTotal = this._measuredValues.tests[suiteName].total; - if (suiteTotal === 0) - throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`); + _prepareMetrics(suite) { + this._measuredValues.tests[suite.name] = { + tests: {}, + total: 0, + }; } async _prepareSuite(suite) { @@ -512,14 +583,22 @@ export class BenchmarkRunner { }); } + _recordPrepareMetric(suite, prepareTime) { + const suiteResults = this._measuredValues.tests[suite.name]; + suiteResults.tests.Prepare = prepareTime; + suiteResults.total += prepareTime; + } + async _runTestAndRecordResults(suite, test) { /* eslint-disable-next-line no-async-promise-executor */ if (this._client?.willRunTest) await this._client.willRunTest(suite, test); // Prepare all mark labels outside the measuring loop. - const startLabel = `${suite.name}.${test.name}-start`; + const syncLabel = `${suite.name}.${test.name}`; + const syncStartLabel = `${suite.name}.${test.name}-start`; const syncEndLabel = `${suite.name}.${test.name}-sync-end`; + const asyncLabel = `${suite.name}.${test.name}-async`; const asyncStartLabel = `${suite.name}.${test.name}-async-start`; const asyncEndLabel = `${suite.name}.${test.name}-async-end`; @@ -528,22 +607,22 @@ export class BenchmarkRunner { let asyncTime; const runSync = () => { if (params.warmupBeforeSync) { - performance.mark("warmup-start"); + this.markStart("warmup", "warmup-start"); const startTime = performance.now(); // Infinite loop for the specified ms. while (performance.now() - startTime < params.warmupBeforeSync) continue; - performance.mark("warmup-end"); + this.markEnd("warmup", "warmup-start", "warmup-end"); } - performance.mark(startLabel); + this.markStart(syncLabel, syncStartLabel); const syncStartTime = performance.now(); test.run(this._page); const syncEndTime = performance.now(); - performance.mark(syncEndLabel); + this.markEnd(syncLabel, syncStartLabel, syncEndLabel); syncTime = syncEndTime - syncStartTime; - performance.mark(asyncStartLabel); + this.markStart(asyncLabel, asyncStartLabel); asyncStartTime = performance.now(); }; const measureAsync = () => { @@ -553,11 +632,7 @@ export class BenchmarkRunner { const asyncEndTime = performance.now(); asyncTime = asyncEndTime - asyncStartTime; this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination. - performance.mark(asyncEndLabel); - if (params.warmupBeforeSync) - performance.measure("warmup", "warmup-start", "warmup-end"); - performance.measure(`${suite.name}.${test.name}-sync`, startLabel, syncEndLabel); - performance.measure(`${suite.name}.${test.name}-async`, asyncStartLabel, asyncEndLabel); + this.markEnd(asyncLabel, asyncStartLabel, asyncEndLabel); }; const report = () => this._recordTestResults(suite, test, syncTime, asyncTime); const invokerClass = params.measurementMethod === "raf" ? RAFTestInvoker : TimerTestInvoker; @@ -568,13 +643,18 @@ export class BenchmarkRunner { async _recordTestResults(suite, test, syncTime, asyncTime) { // Skip reporting updates for the warmup suite. - if (suite === WarmupSuite) + if (suite.name === "Warmup") return; - const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 }; + const suiteResults = this._measuredValues.tests[suite.name]; const total = syncTime + asyncTime; - this._measuredValues.tests[suite.name] = suiteResults; - suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; + suiteResults.tests[test.name] = { + tests: { + Sync: syncTime, + Async: asyncTime, + }, + total: total + }; suiteResults.total += total; if (this._client?.didRunTest) @@ -611,16 +691,24 @@ export class BenchmarkRunner { throw new Error(`Requested iteration=${i} does not exist.`); return getMetric(`Iteration-${i}-Total`); }; + const aggregates = Object.create(null); const collectSubMetrics = (prefix, items, parent) => { for (let name in items) { const results = items[name]; const metric = getMetric(prefix + name); - metric.add(results.total ?? results); + const value = results.total ?? results; + metric.add(value); if (metric.parent !== parent) parent.addChild(metric); - if (results.tests) + if (results.tests) { collectSubMetrics(`${metric.name}${Metric.separator}`, results.tests, metric); + } else { + let aggregate = aggregates[name]; + if (!aggregate) + aggregate = aggregates[name] = new Metric(name); + aggregate.add(value); + } } }; const initializeMetrics = this._metrics === null; @@ -643,6 +731,10 @@ export class BenchmarkRunner { const iterationTotal = iterationTotalMetric(geomean.length); for (const results of Object.values(iterationResults)) iterationTotal.add(results.total); + for (let aggregate of Object.values(aggregates)) { + aggregate.computeAggregatedMetrics(); + getMetric(aggregate.name).add(aggregate.geomean); + } iterationTotal.computeAggregatedMetrics(); geomean.add(iterationTotal.geomean); getMetric("Score").add(geomeanToScore(iterationTotal.geomean)); @@ -651,3 +743,145 @@ export class BenchmarkRunner { metric.computeAggregatedMetrics(); } } + +class PingBackDone {} +const DONE = new PingBackDone(); + +class PostMessageRunner extends BenchmarkRunner { + _postMessageHandlers = { __proto__: null }; + + constructor(suites, client) { + super(suites, client); + globalThis.addEventListener("message", this.onMessage.bind(this)); + } + + onMessage(event) { + const { cmd, data = undefined } = event.data; + const resolver = this._postMessageHandlers[cmd]; + if (!resolver) + throw Error(`Unknown onMessage: ${cmd}`); + if (resolver === DONE) + throw Error(`Cannot evaluate ping back resolver twice: ${cmd}`); + this._postMessageHandlers[cmd] = DONE; + resolver(data); + } + + async pingBackMessage(name) { + if (this._postMessageHandlers[name] !== DONE) + throw Error(`Unknown ping back message: ${name}`); + return new Promise(resolver => { + this._postMessageHandlers[name] = resolver; + }); + } +} + +export class SubdomainBenchmarkRunner extends PostMessageRunner { + + _finishPromise; + _postMessageHandlers = { + __proto__: null, + prepareDone: DONE, + recordPrepareMetric: DONE, + recordTestResults: DONE, + }; + + // Assume all workloads report the metrics back via postMessage. + async _runSuite(suite) { + this._prepareMetrics(suite); + await this._prepareSuite(suite); + if (params.measurePrepare) { + const { prepareTime } = await this.pingBackMessage("recordPrepareMetric"); + this._recordPrepareMetric(suite, prepareTime); + } + for (const test of suite.tests) + await this._runTestAndRecordResults(suite, test); + } + + async _runTestAndRecordResults(suite, test) { + this.postMessage("runTestAndRecordResults"); + const { syncTime, asyncTime } = await this.pingBackMessage("recordTestResults"); + await this._recordTestResults(suite, test, syncTime, asyncTime); + } + + postMessage(cmd, data) { + this._frame.contentWindow.postMessage({ cmd, data }, "*"); + } + + async _prepareSuite(suite) { + await new Promise((resolve) => { + const frame = this._page._frame; + frame.onload = async () => resolve(); + const url = new URL(window.location.href); + const customParams = params.copy(); + customParams.embedded = true; + customParams.suites = [suite.name]; + customParams.tags = []; + customParams.startAutomatically = true; + customParams.developerMode = false; + customParams.iterationCount = 1; + // TODO: handle local test URLs better + if (this._isLocalUrl(url)) { + url.port = parseInt(url.port) + this._currentIteration; + } else { + // Input: foo-bar.custom.domain.com, foo-bar-0.custom.domain.com + // Output: foo-bar-1.custom.domain.com, foo-bar-1.custom.domain.com + const hostParts = url.hostname.split("."); + const domainWithoutIndex = hostParts[0].match(DOMAIN_WITHOUT_INDEX_RE).groups.nonIndexed; + hostParts[0] = `${domainWithoutIndex}-${this.currentIteration}`; + url.host = hostParts.join("."); + } + url.search = customParams.toSearchParams(); + frame.src = url.toString(); + }); + await this.pingBackMessage("prepareDone"); + this.postMessage("startSuite"); + } + + _isLocalUrl(url) { + return url.hostname === "127.0.0.1" || url.hostname === "localhost"; + } +} + +const DOMAIN_WITHOUT_INDEX_RE = /(?([^-]+(-[^0-9])?)+)(-[0-9]+)?/; + +export class EmbeddedBenchmarkRunner extends PostMessageRunner { + + _postMessageHandlers = { + __proto__: null, + startSuite: DONE, + runTestAndRecordResults: DONE, + }; + + constructor(suites, client) { + super(suites, client); + if (globalThis === globalThis.parent) { + const msg = "Cannot use embedded runner in toplevel iframe"; + alert(msg); + throw Error(msg); + } + } + + postMessage(cmd, data) { + globalThis.parent.postMessage({ cmd, data }, "*"); + } + + async _prepareSuite(suite) { + await super._prepareSuite(suite); + this.postMessage("prepareDone", { prepareTime: 0 }); + await this.pingBackMessage("startSuite"); + } + + async _runTestAndRecordResults(suite, test) { + await this.pingBackMessage("runTestAndRecordResults"); + return super._runTestAndRecordResults(suite, test); + } + + _recordPrepareMetric(suite, prepareTime) { + super._recordPrepareMetric(suite, prepareTime); + this.postMessage("recordPrepareMetric", { prepareTime }); + } + + async _recordTestResults(suite, test, syncTime, asyncTime) { + this.postMessage("recordTestResults", { syncTime, asyncTime }); + } +} diff --git a/resources/developer-mode.mjs b/resources/developer-mode.mjs index 4def90e..24a80c3 100644 --- a/resources/developer-mode.mjs +++ b/resources/developer-mode.mjs @@ -23,8 +23,10 @@ export function createDeveloperModeContainer() { settings.append(createUIForWarmupBeforeSync()); settings.append(createUIForSyncStepDelay()); settings.append(createUIForLeakyIframes()); + settings.append(createUIForMeasurePrepare()); + settings.append(createUIForDomainPerIteration()); + settings.append(createUIForForceGC()); - content.append(document.createElement("hr")); content.append(settings); content.append(document.createElement("hr")); @@ -92,6 +94,57 @@ function createUIForWarmupSuite() { return label; } +function createUIForMeasurePrepare() { + let check = document.createElement("input"); + check.type = "checkbox"; + check.id = "measure-prepare"; + check.checked = params.measurePrepare; + + check.onchange = () => { + params.measurePrepare = check.checked; + updateURL(); + }; + + let label = document.createElement("label"); + label.append(check, " ", "Measure Prepare"); + + return label; +} + +function createUIForDomainPerIteration() { + let check = document.createElement("input"); + check.type = "checkbox"; + check.id = "domain-per-iteration"; + check.checked = params.domainPerIteration; + + check.onchange = () => { + params.domainPerIteration = check.checked; + updateURL(); + }; + + let label = document.createElement("label"); + label.append(check, " ", "Use Subdomain-Runner"); + + return label; +} + +function createUIForForceGC() { + let check = document.createElement("input"); + check.type = "checkbox"; + check.id = "suite-force-gc"; + check.checked = params.suiteForceGC; + + check.onchange = () => { + params.suiteForceGC = check.checked; + updateURL(); + }; + + let label = document.createElement("label"); + label.append(check, " ", "Force GC before every Suite"); + + return label; +} + function createUIForIterationCount() { const { range, label } = createTimeRangeUI("Iterations: ", params.iterationCount, "#", 1, 200); range.onchange = () => { @@ -291,7 +344,7 @@ function updateURL() { else url.searchParams.delete("measurementMethod"); - const boolParamKeys = ["iterationCount", "useWarmupSuite", "warmupBeforeSync", "waitBeforeSync", "leakyIframes"]; + const boolParamKeys = ["iterationCount", "useWarmupSuite", "warmupBeforeSync", "waitBeforeSync", "domainPerIteration", "suiteForceGC", "leakyIframes"]; for (const paramKey of boolParamKeys) { if (params[paramKey] !== defaultParams[paramKey]) url.searchParams.set(paramKey, params[paramKey]); @@ -299,6 +352,11 @@ function updateURL() { url.searchParams.delete(paramKey); } + if (params.measurePrepare !== defaultParams.measurePrepare) + url.searchParams.set("measurePrepare", params.measurePrepare); + else + url.searchParams.delete("measurePrepare"); + // Only push state if changed url.search = decodeURIComponent(url.search); if (url.href !== window.location.href) diff --git a/resources/main.css b/resources/main.css index 0ce25e4..2a3a58d 100644 --- a/resources/main.css +++ b/resources/main.css @@ -836,3 +836,21 @@ section#about .note { color: #fff; stroke: #fff; } + + +.embedded section { + border: none; + position: absolute; + top: 0; + left: 0; + margin: 0; + padding: 0; +} + +.embedded #logo, +.embedded #progress, +.embedded #info, +.embedded #info-progress, +.embedded #info-label { + display: none !important; +} \ No newline at end of file diff --git a/resources/main.mjs b/resources/main.mjs index c406205..0158803 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -1,4 +1,4 @@ -import { BenchmarkRunner } from "./benchmark-runner.mjs"; +import { BenchmarkRunner, SubdomainBenchmarkRunner, EmbeddedBenchmarkRunner } from "./benchmark-runner.mjs"; import * as Statistics from "./statistics.mjs"; import { Suites } from "./tests.mjs"; import { renderMetricView } from "./metric-ui.mjs"; @@ -12,11 +12,13 @@ class MainBenchmarkClient { suitesCount = null; _measuredValuesList = []; _finishedTestCount = 0; + _totalSubtestsCount = 0; _progressCompleted = null; _isRunning = false; _hasResults = false; _developerModeContainer = null; _metrics = Object.create(null); + _runner = null; constructor() { window.addEventListener("DOMContentLoaded", () => this.prepareUI()); @@ -60,14 +62,19 @@ class MainBenchmarkClient { this._isRunning = true; const enabledSuites = Suites.filter((suite) => !suite.disabled); - const totalSubtestsCount = enabledSuites.reduce((testsCount, suite) => { + this._totalSubtestsCount = enabledSuites.reduce((testsCount, suite) => { return testsCount + suite.tests.length; }, 0); - this.stepCount = params.iterationCount * totalSubtestsCount; + this.stepCount = params.iterationCount * this._totalSubtestsCount; this._progressCompleted.max = this.stepCount; this.suitesCount = enabledSuites.length; - const runner = new BenchmarkRunner(Suites, this); - runner.runMultipleIterations(params.iterationCount); + if (params.embedded) + this._runner = new EmbeddedBenchmarkRunner(Suites, this); + else if (params.domainPerIteration) + this._runner = new SubdomainBenchmarkRunner(Suites, this); + else + this._runner = new BenchmarkRunner(Suites, this); + this._runner.runMultipleIterations(params.iterationCount); return true; } @@ -76,9 +83,6 @@ class MainBenchmarkClient { } willAddTestFrame(frame) { - frame.style.left = "50%"; - frame.style.top = "50%"; - frame.style.transform = "translate(-50%, -50%)"; } willRunTest(suite, test) { @@ -97,7 +101,7 @@ class MainBenchmarkClient { willStartFirstIteration() { this._measuredValuesList = []; - this._finishedTestCount = 0; + this._finishedTestCount = params.currentIteration * this._totalSubtestsCount; } didFinishLastIteration(metrics) { @@ -105,6 +109,9 @@ class MainBenchmarkClient { this._isRunning = false; this._hasResults = true; this._metrics = metrics; + // Don't display the results if the runnier is embedded. + if (params.embedded) + return; const scoreResults = this._computeResults(this._measuredValuesList, "score"); if (scoreResults.isValid) @@ -213,8 +220,9 @@ class MainBenchmarkClient { const trackHeight = 24; document.documentElement.style.setProperty("--metrics-line-height", `${trackHeight}px`); const plotWidth = (params.viewport.width - 120) / 2; - document.getElementById("geomean-chart").innerHTML = renderMetricView({ - metrics: [metrics.Geomean], + const singleToplevelMetrics = Object.values(metrics).filter((each) => each && each.unit === "ms" && !each.parent && !each.children.length); + document.getElementById("toplevel-chart").innerHTML = renderMetricView({ + metrics: singleToplevelMetrics, width: plotWidth, trackHeight, renderChildren: false, @@ -258,6 +266,8 @@ class MainBenchmarkClient { } prepareUI() { + if (params.embedded) + document.body.className = "embedded"; window.addEventListener("hashchange", this._hashChangeHandler.bind(this)); window.addEventListener("resize", this._resizeScreeHandler.bind(this)); this._resizeScreeHandler(); diff --git a/resources/metric.mjs b/resources/metric.mjs index 66830ee..4d769ff 100644 --- a/resources/metric.mjs +++ b/resources/metric.mjs @@ -5,7 +5,7 @@ export const MILLISECONDS_PER_MINUTE = 60 * 1000; export class Metric { static separator = "/"; - constructor(name, unit = "ms") { + constructor(name, unit = "ms", values = []) { if (typeof name !== "string") throw new Error(`Invalid metric.name=${name}, expected string.`); this.name = name; @@ -26,11 +26,17 @@ export class Metric { this.min = 0.0; this.max = 0.0; - this.values = []; + this.values = values; // Mark properties which refer to other Metric objects as // non-enumerable to avoid issue with JSON.stringify due to circular // references. + Object.defineProperties(this, { + children: { + writable: true, + value: [], + }, + }); Object.defineProperties(this, { parent: { writable: true, diff --git a/resources/params.mjs b/resources/params.mjs index 6ade9d4..726b2af 100644 --- a/resources/params.mjs +++ b/resources/params.mjs @@ -1,4 +1,4 @@ -class Params { +export class Params { viewport = { width: 800, height: 600, @@ -27,11 +27,19 @@ class Params { shuffleSeed = "off"; // Leak iframes to DevTools can display sources. leakyIframes = false; + // Emit metrics for the prepare phase. + measurePrepare = false; + // Use a new subdomain for each iteration to avoid caching. + domainPerIteration = false; + // Force a gc before every suite (chrome specific) + suiteForceGC = false; + currentIteration = 0; + embedded = false; - constructor(searchParams = undefined) { + constructor(searchParams = undefined, freeze = true) { if (searchParams) this._copyFromSearchParams(searchParams); - if (!this.developerMode) { + if (!this.developerMode && freeze) { Object.freeze(this.viewport); Object.freeze(this); } @@ -65,24 +73,44 @@ class Params { if (searchParams.has("suite") || searchParams.has("suites")) { if (searchParams.has("suite") && searchParams.has("suites")) throw new Error("Params 'suite' and 'suites' can not be used together."); - const value = searchParams.get("suite") || searchParams.get("suites"); - this.suites = value.split(","); - if (this.suites.length === 0) - throw new Error("No suites selected"); + const value = (searchParams.get("suite") || searchParams.get("suites")).trim(); + if (value.length ) { + this.suites = value.split(","); + if (this.suites.length === 0) + throw new Error("No suites selected"); + } searchParams.delete("suite"); searchParams.delete("suites"); } if (searchParams.has("tags")) { - if (this.suites.length) - throw new Error("'suites' and 'tags' cannot be used together."); - this.tags = searchParams.get("tags").split(","); + const rawTags = searchParams.get("tags"); + if (rawTags) { + this.tags = rawTags.split(","); + if (this.tags.length && this.suites.length) + throw new Error("'suites' and 'tags' cannot be used together."); + } searchParams.delete("tags"); } + this.embedded = searchParams.has("embedded"); + searchParams.delete("embedded"); + this.developerMode = searchParams.has("developerMode"); searchParams.delete("developerMode"); + this.measurePrepare = searchParams.has("measurePrepare"); + searchParams.delete("measurePrepare"); + + this.domainPerIteration = searchParams.has("domainPerIteration"); + searchParams.delete("domainPerIteration"); + + this.suiteForceGC = searchParams.has("suiteForceGC"); + searchParams.delete("suiteForceGC"); + + this.currentIteration = searchParams.get("currentIteration") | 0; + searchParams.delete("currentIteration"); + if (searchParams.has("useWarmupSuite")) { this.useWarmupSuite = true; searchParams.delete("useWarmupSuite"); @@ -134,10 +162,26 @@ class Params { console.error("Got unused search params", unused); } + toSearchParamsObject() { + const rawParams = { __proto__: null }; + for (const [key, value] of Object.entries(this)) { + if (value === defaultParams[key]) + continue; + rawParams[key] = value; + } + // Either suites or params can be used at the same time. + if (rawParams.suites?.length && rawParams.tags?.length) + delete rawParams.suites; + rawParams.viewport = `${this.viewport.width}x${this.viewport.height}`; + return new URLSearchParams(rawParams); + } + toSearchParams() { - const rawParams = { ...this }; - rawParams["viewport"] = `${this.viewport.width}x${this.viewport.height}`; - return new URLSearchParams(rawParams).toString(); + return this.toSearchParamsObject().toString(); + } + + copy() { + return new Params(this.toSearchParamsObject(), false); } } @@ -152,3 +196,4 @@ try { alert(`Invalid URL Param: ${e}`); } export const params = maybeCustomParams; +globalThis.params = maybeCustomParams; diff --git a/resources/tests.mjs b/resources/tests.mjs index 625ae4f..7304275 100644 --- a/resources/tests.mjs +++ b/resources/tests.mjs @@ -304,6 +304,39 @@ Suites.push({ ], }); +// Downstream only. +Suites.push({ + name: "TodoMVC-React-Window", + url: "todomvc/architecture-examples/react/dist/index-window.html#/home", + tags: ["todomvc"], + disabled: true, + async prepare(page) { + await page.targetChildWindow(); + const element = await page.waitForElement(".new-todo"); + element.focus(); + }, + tests: [ + new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => { + const newTodo = page.querySelector(".new-todo"); + for (let i = 0; i < numberOfItemsToAdd; i++) { + newTodo.setValue(getTodoText(defaultLanguage, i)); + newTodo.dispatchEvent("input"); + newTodo.enter("keydown"); + } + }), + new BenchmarkTestStep("CompletingAllItems", (page) => { + const checkboxes = page.querySelectorAll(".toggle"); + for (let i = 0; i < numberOfItemsToAdd; i++) + checkboxes[i].click(); + }), + new BenchmarkTestStep("DeletingAllItems", (page) => { + const deleteButtons = page.querySelectorAll(".destroy"); + for (let i = numberOfItemsToAdd - 1; i >= 0; i--) + deleteButtons[i].click(); + }), + ], +}); + Suites.push({ name: "TodoMVC-React-Redux", url: "todomvc/architecture-examples/react-redux/dist/index.html", @@ -1036,6 +1069,7 @@ Suites.push({ ], }); + Object.freeze(Suites); Suites.forEach((suite) => { if (!suite.tags) diff --git a/resources/todomvc/architecture-examples/react/README.md b/resources/todomvc/architecture-examples/react/README.md index f500180..aad05b2 100644 --- a/resources/todomvc/architecture-examples/react/README.md +++ b/resources/todomvc/architecture-examples/react/README.md @@ -6,6 +6,8 @@ This application uses React 17.0.2 to implement a todo application. - [React](https://reactjs.org/) is a JavaScript library for creating user interfaces. +The downstream version also generates the TodoMVC-React-Window workload. A variation of TodoMVC-React that renders and controls the app in a new window using the scripts loaded in the parent window. + ## Implementation details React:\ @@ -22,6 +24,8 @@ Controller: serves as an interface between view & model components To build the static files, this application utilizes webpack. It minifies and optimizes output files and copies all necessary files to a `dist` folder. +For TodoMVC-React-Window, the webpack scripts separate the JavaScript from the CSS. The JavaScript is loaded by index.html in the main window, while the css is loaded by todo.html in the child window. + ## Requirements The only requirement is an installation of Node, to be able to install dependencies and run scripts to serve a local server. diff --git a/resources/todomvc/architecture-examples/react/dist/index-window.html b/resources/todomvc/architecture-examples/react/dist/index-window.html new file mode 100644 index 0000000..f5d5894 --- /dev/null +++ b/resources/todomvc/architecture-examples/react/dist/index-window.html @@ -0,0 +1 @@ +TodoMVC: React-Window \ No newline at end of file diff --git a/resources/todomvc/architecture-examples/react/dist/todo.html b/resources/todomvc/architecture-examples/react/dist/todo.html new file mode 100644 index 0000000..a6e2a16 --- /dev/null +++ b/resources/todomvc/architecture-examples/react/dist/todo.html @@ -0,0 +1 @@ +TodoMVC: React-Window inner window

Click on input field to write your todo.

At least two characters are needed to be a valid entry.

Press 'enter' to add the todo.

Double-click to edit a todo

\ No newline at end of file diff --git a/resources/todomvc/architecture-examples/react/dist/windowed.bundle.js b/resources/todomvc/architecture-examples/react/dist/windowed.bundle.js new file mode 100644 index 0000000..f71f776 --- /dev/null +++ b/resources/todomvc/architecture-examples/react/dist/windowed.bundle.js @@ -0,0 +1,3 @@ +/*! For license information please see windowed.bundle.js.LICENSE.txt */ +(()=>{var e,t,n={184:(e,t)=>{var n;!function(){"use strict";var r={}.hasOwnProperty;function l(){for(var e=[],t=0;t{"use strict";var t=Object.getOwnPropertySymbols,n=Object.prototype.hasOwnProperty,r=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map((function(e){return t[e]})).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach((function(e){r[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},r)).join("")}catch(e){return!1}}()?Object.assign:function(e,l){for(var a,o,i=function(e){if(null==e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}(e),u=1;u{"use strict";var r=n(294),l=n(418),a=n(840);function o(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n