Merge v3.0-custom changes to v3.1-custom

- TodoMVC-React windowed variation
- Add signpost support
- Add suiteForceGC helper
- Subdomain running options:
  - hosted: using `-$ITER` subdomain suffix
  - local: using increasing port numbers
- Add measure prepare phase for "loading" style metric
- Move the warmup suite to the main config
- Display all top level metrics directly in the details view

Change-Id: Ib813e8d83fd164fb2849b6889f187c1904b21965

.eslintignore

@@ -23,6 +23,8 @@ package-lock.json
 !/resources/todomvc/architecture-examples/vue
 !/resources/todomvc/architecture-examples/vue-complex
 
+/resources/todomvc/architecture-examples/react/public/window/*
+
 /resources/todomvc/vanilla-examples/*
 !/resources/todomvc/vanilla-examples/javascript-es5
 !/resources/todomvc/vanilla-examples/javascript-es5-complex

.prettierignore

@@ -23,6 +23,8 @@ package-lock.json
 !/resources/todomvc/architecture-examples/vue
 !/resources/todomvc/architecture-examples/vue-complex
 
+/resources/todomvc/architecture-examples/react/public/window/*
+
 /resources/todomvc/vanilla-examples/*
 !/resources/todomvc/vanilla-examples/javascript-es5
 !/resources/todomvc/vanilla-examples/javascript-es5-complex

index.html

@@ -86,6 +86,8 @@
                             <h2>Aggregate Metric</h2>
                             <div id="geomean-chart"></div>
                             <h2>Test Metrics Overview</h2>
+                            <div id="toplevel-chart"></div>
+                            <h2>Tests</h2>
                             <div id="tests-chart"></div>
                         </div>
                         <br />

resources/benchmark-runner.mjs

@@ -22,6 +22,9 @@ function getParent(lookupStartNode, path) {
 class Page {
     constructor(frame) {
         this._frame = frame;
+        // Downstream-only. this._lookupStartNode is used by the React-Window workload to run the
+        // querySelector* methods on a child window.
+        this._lookupStartNode = null;
     }
 
     layout() {
@@ -58,7 +61,7 @@ class Page {
      * @returns PageElement | null
      */
     querySelector(selector, path = []) {
-        const lookupStartNode = this._frame.contentDocument;
+        const lookupStartNode = this._lookupStartNode ?? this._frame.contentDocument;
         const element = getParent(lookupStartNode, path).querySelector(selector);
 
         if (element === null)
@@ -82,7 +85,7 @@ class Page {
      * @returns array
      */
     querySelectorAll(selector, path = []) {
-        const lookupStartNode = this._frame.contentDocument;
+        const lookupStartNode = this._lookupStartNode ?? this._frame.contentDocument;
         const elements = Array.from(getParent(lookupStartNode, path).querySelectorAll(selector));
         for (let i = 0; i < elements.length; i++)
             elements[i] = this._wrapElement(elements[i]);
@@ -114,6 +117,24 @@ class Page {
     _wrapElement(element) {
         return new PageElement(element);
     }
+
+    // Downstream only.
+    async targetChildWindow() {
+        return new Promise((resolve) => {
+            const resolveIfReady = () => {
+                // Expects the workload to set a global childWindow object.
+                const maybeChildWindow = this._frame.contentWindow.globalThis.childWindow;
+                let callback = resolveIfReady;
+                if (maybeChildWindow) {
+                    this._lookupStartNode = maybeChildWindow.document;
+                    resolve();
+                    return;
+                }
+                window.requestAnimationFrame(callback);
+            };
+            resolveIfReady();
+        });
+    }
 }
 
 const NATIVE_OPTIONS = {
@@ -218,9 +239,10 @@ function geomeanToScore(geomean) {
 // The WarmupSuite is used to make sure all runner helper functions and
 // classes are compiled, to avoid unnecessary pauses due to delayed
 // compilation of runner methods in the middle of the measuring cycle.
-const WarmupSuite = {
+export const WarmupSuite = {
     name: "Warmup",
     url: "warmup/index.html",
+    disabled: false,
     async prepare(page) {
         await page.waitForElement("#testItem");
     },
@@ -355,15 +377,48 @@ export class BenchmarkRunner {
         this._page = null;
         this._metrics = null;
         this._iterationCount = params.iterationCount;
+        this._currentIteration = 0;
         if (params.shuffleSeed !== "off")
             this._suiteOrderRandomNumberGenerator = seededHashRandomNumberGenerator(params.shuffleSeed);
         this._wakeLock = new WakeLock();
+        this._maybeForceGC = this._initForceGC();
+        this._useSignPost = !!globalThis.__signpostStart;
+    }
+
+    markStart(groupLabel, startLabel) {
+        performance.mark(startLabel);
+        if (this._useSignPost)
+            globalThis.__signpostStart(groupLabel);
+    }
+
+    markEnd(groupLabel, startLabel, endLabel) {
+        performance.mark(endLabel);
+        if (this._useSignPost)
+            globalThis.__signpostStop(groupLabel);
+        performance.measure(groupLabel, startLabel, endLabel);
+    }
+
+    _initForceGC() {
+        if (!params.suiteForceGC)
+            return () => {};
+        try {
+            let gc_extension = eval("gc");
+            return () => gc_extension({ type: "major", execution: "async" });
+        } catch (e) {
+            console.error("suiteForceGC was enabled but --expose-gc was not set", e);
+        }
+        return () => {};
+    }
+
+    get currentIteration() {
+        return this._currentIteration;
     }
 
     async runMultipleIterations(iterationCount) {
         this._iterationCount = iterationCount;
+        this._currentIteration = 0;
         if (this._client?.willStartFirstIteration)
-            await this._client.willStartFirstIteration(iterationCount);
+            await this._client.willStartFirstIteration( this._iterationCount);
 
         try {
             await this._runMultipleIterations();
@@ -382,25 +437,29 @@ export class BenchmarkRunner {
     async _runMultipleIterations() {
         const iterationStartLabel = "iteration-start";
         const iterationEndLabel = "iteration-end";
-        for (let i = 0; i < this._iterationCount; i++) {
-            performance.mark(iterationStartLabel);
+        for (; this._currentIteration < this._iterationCount; this._currentIteration++) {
+            const iterationLabel = `iteration-${this._currentIteration}`;
+            this.markStart(iterationLabel, iterationStartLabel);
             await this._runAllSuites();
-            performance.mark(iterationEndLabel);
-            performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel);
+            this.markEnd(iterationLabel, iterationStartLabel, iterationEndLabel);
         }
     }
 
     _removeFrame() {
         if (this._frame) {
-            if (params.leakyIframes)
+            if (params.leakyIframes) {
                 document.getElementById("leakedIframes").append(this._frame);
-            else
+            } else {
+                // Downstream only. The React-Window workload sets a global childWindow object.
+                if (this._frame.contentWindow.globalThis.childWindow)
+                    this._frame.contentWindow.globalThis.childWindow.close();
                 this._frame.parentNode.removeChild(this._frame);
+            }
             this._frame = null;
         }
     }
 
-    async _appendFrame(src) {
+    async _appendFrame() {
         const frame = document.createElement("iframe");
         const style = frame.style;
         style.width = `${params.viewport.width}px`;
@@ -409,9 +468,14 @@ export class BenchmarkRunner {
         style.position = "absolute";
         frame.setAttribute("scrolling", "no");
         frame.className = "test-runner";
-        style.left = "50%";
-        style.top = "50%";
-        style.transform = "translate(-50%, -50%)";
+        if (params.embedded) {
+            style.left = "0px";
+            style.top = "0px";
+        } else {
+            style.left = "50%";
+            style.top = "50%";
+            style.transform = "translate(-50%, -50%)";
+        }
 
         if (this._client?.willAddTestFrame)
             await this._client.willAddTestFrame(frame);
@@ -425,10 +489,11 @@ export class BenchmarkRunner {
         this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
         await this._wakeLock.request();
 
+        const prepareLabel = "runner-prepare";
         const prepareStartLabel = "runner-prepare-start";
         const prepareEndLabel = "runner-prepare-end";
 
-        performance.mark(prepareStartLabel);
+        this.markStart(prepareLabel, prepareStartLabel);
         this._removeFrame();
         await this._appendFrame();
         this._page = new Page(this._frame);
@@ -444,8 +509,7 @@ export class BenchmarkRunner {
                 suites[j] = tmp;
             }
         }
-        performance.mark(prepareEndLabel);
-        performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel);
+        this.markEnd(prepareLabel, prepareStartLabel, prepareEndLabel);
 
         try {
             for (const suite of suites) {
@@ -459,46 +523,53 @@ export class BenchmarkRunner {
     }
 
     async _finishRunAllSuites() {
+        const finalizeLabel = "runner-finalize";
         const finalizeStartLabel = "runner-finalize-start";
         const finalizeEndLabel = "runner-finalize-end";
 
-        performance.mark(finalizeStartLabel);
+        this.markStart(finalizeLabel, finalizeStartLabel);
         // Remove frame to clear the view for displaying the results.
         this._removeFrame();
         await this._finalize();
-        performance.mark(finalizeEndLabel);
-        performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel);
+        this.markEnd(finalizeLabel, finalizeStartLabel, finalizeEndLabel);
         await this._wakeLock.release();
     }
 
     async _runSuite(suite) {
-        const suiteName = suite.name;
-        const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
-        const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
-        const suiteStartLabel = `suite-${suiteName}-start`;
-        const suiteEndLabel = `suite-${suiteName}-end`;
+        this._prepareMetrics(suite);
+        const suitePrepareLabel = `suite-${suite.name}-prepare`;
+        const suitePrepareStartLabel = `suite-${suite.name}-prepare-start`;
+        const suitePrepareEndLabel = `suite-${suite.name}-prepare-end`;
+        const suiteLabel = `suite-${suite.name}`;
+        const suiteStartLabel = `suite-${suite.name}-start`;
+        const suiteEndLabel = `suite-${suite.name}-end`;
 
-        performance.mark(suitePrepareStartLabel);
+        await this._maybeForceGC();
+
+        this.markStart(suitePrepareLabel, suitePrepareStartLabel);
+        const prepareStartTime = performance.now();
         await this._prepareSuite(suite);
-        performance.mark(suitePrepareEndLabel);
+        const prepareEndTime = performance.now();
+        this.markEnd(suitePrepareLabel, suitePrepareStartLabel, suitePrepareEndLabel);
 
-        performance.mark(suiteStartLabel);
+        await this._maybeForceGC();
+
+        if (params.measurePrepare) {
+            const prepareTime = prepareEndTime - prepareStartTime;
+            this._recordPrepareMetric(suite, prepareTime);
+        }
+
+        this.markStart(suiteLabel, suiteStartLabel);
         for (const test of suite.tests)
             await this._runTestAndRecordResults(suite, test);
-        performance.mark(suiteEndLabel);
-
-        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
-        performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
-        this._validateSuiteTotal(suiteName);
+        this.markEnd(suiteLabel, suiteStartLabel, suiteEndLabel);
     }
 
-    _validateSuiteTotal(suiteName) {
-        // When the test is fast and the precision is low (for example with Firefox'
-        // privacy.resistFingerprinting preference), it's possible that the measured
-        // total duration for an entire is 0.
-        const suiteTotal = this._measuredValues.tests[suiteName].total;
-        if (suiteTotal === 0)
-            throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`);
+    _prepareMetrics(suite) {
+        this._measuredValues.tests[suite.name] = {
+            tests: {},
+            total: 0,
+        };
     }
 
     async _prepareSuite(suite) {
@@ -512,14 +583,22 @@ export class BenchmarkRunner {
         });
     }
 
+    _recordPrepareMetric(suite, prepareTime) {
+        const suiteResults = this._measuredValues.tests[suite.name];
+        suiteResults.tests.Prepare = prepareTime;
+        suiteResults.total += prepareTime;
+    }
+
     async _runTestAndRecordResults(suite, test) {
         /* eslint-disable-next-line no-async-promise-executor */
         if (this._client?.willRunTest)
             await this._client.willRunTest(suite, test);
 
         // Prepare all mark labels outside the measuring loop.
-        const startLabel = `${suite.name}.${test.name}-start`;
+        const syncLabel = `${suite.name}.${test.name}`;
+        const syncStartLabel = `${suite.name}.${test.name}-start`;
         const syncEndLabel = `${suite.name}.${test.name}-sync-end`;
+        const asyncLabel = `${suite.name}.${test.name}-async`;
         const asyncStartLabel = `${suite.name}.${test.name}-async-start`;
         const asyncEndLabel = `${suite.name}.${test.name}-async-end`;
 
@@ -528,22 +607,22 @@ export class BenchmarkRunner {
         let asyncTime;
         const runSync = () => {
             if (params.warmupBeforeSync) {
-                performance.mark("warmup-start");
+                this.markStart("warmup", "warmup-start");
                 const startTime = performance.now();
                 // Infinite loop for the specified ms.
                 while (performance.now() - startTime < params.warmupBeforeSync)
                     continue;
-                performance.mark("warmup-end");
+                this.markEnd("warmup", "warmup-start", "warmup-end");
             }
-            performance.mark(startLabel);
+            this.markStart(syncLabel, syncStartLabel);
             const syncStartTime = performance.now();
             test.run(this._page);
             const syncEndTime = performance.now();
-            performance.mark(syncEndLabel);
+            this.markEnd(syncLabel, syncStartLabel, syncEndLabel);
 
             syncTime = syncEndTime - syncStartTime;
 
-            performance.mark(asyncStartLabel);
+            this.markStart(asyncLabel, asyncStartLabel);
             asyncStartTime = performance.now();
         };
         const measureAsync = () => {
@@ -553,11 +632,7 @@ export class BenchmarkRunner {
             const asyncEndTime = performance.now();
             asyncTime = asyncEndTime - asyncStartTime;
             this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
-            performance.mark(asyncEndLabel);
-            if (params.warmupBeforeSync)
-                performance.measure("warmup", "warmup-start", "warmup-end");
-            performance.measure(`${suite.name}.${test.name}-sync`, startLabel, syncEndLabel);
-            performance.measure(`${suite.name}.${test.name}-async`, asyncStartLabel, asyncEndLabel);
+            this.markEnd(asyncLabel, asyncStartLabel, asyncEndLabel);
         };
         const report = () => this._recordTestResults(suite, test, syncTime, asyncTime);
         const invokerClass = params.measurementMethod === "raf" ? RAFTestInvoker : TimerTestInvoker;
@@ -568,13 +643,18 @@ export class BenchmarkRunner {
 
     async _recordTestResults(suite, test, syncTime, asyncTime) {
         // Skip reporting updates for the warmup suite.
-        if (suite === WarmupSuite)
+        if (suite.name === "Warmup")
             return;
 
-        const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 };
+        const suiteResults = this._measuredValues.tests[suite.name];
         const total = syncTime + asyncTime;
-        this._measuredValues.tests[suite.name] = suiteResults;
-        suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
+        suiteResults.tests[test.name] = {
+            tests: {
+                Sync: syncTime,
+                Async: asyncTime,
+            },
+            total: total
+        };
         suiteResults.total += total;
 
         if (this._client?.didRunTest)
@@ -611,16 +691,24 @@ export class BenchmarkRunner {
                 throw new Error(`Requested iteration=${i} does not exist.`);
             return getMetric(`Iteration-${i}-Total`);
         };
+        const aggregates = Object.create(null);
 
         const collectSubMetrics = (prefix, items, parent) => {
             for (let name in items) {
                 const results = items[name];
                 const metric = getMetric(prefix + name);
-                metric.add(results.total ?? results);
+                const value = results.total ?? results;
+                metric.add(value);
                 if (metric.parent !== parent)
                     parent.addChild(metric);
-                if (results.tests)
+                if (results.tests) {
                     collectSubMetrics(`${metric.name}${Metric.separator}`, results.tests, metric);
+                } else {
+                    let aggregate = aggregates[name];
+                    if (!aggregate)
+                        aggregate = aggregates[name] = new Metric(name);
+                    aggregate.add(value);
+                }
             }
         };
         const initializeMetrics = this._metrics === null;
@@ -643,6 +731,10 @@ export class BenchmarkRunner {
         const iterationTotal = iterationTotalMetric(geomean.length);
         for (const results of Object.values(iterationResults))
             iterationTotal.add(results.total);
+        for (let aggregate of Object.values(aggregates)) {
+            aggregate.computeAggregatedMetrics();
+            getMetric(aggregate.name).add(aggregate.geomean);
+        }
         iterationTotal.computeAggregatedMetrics();
         geomean.add(iterationTotal.geomean);
         getMetric("Score").add(geomeanToScore(iterationTotal.geomean));
@@ -651,3 +743,145 @@ export class BenchmarkRunner {
             metric.computeAggregatedMetrics();
     }
 }
+
+class PingBackDone {}
+const DONE = new PingBackDone();
+
+class PostMessageRunner extends BenchmarkRunner {
+    _postMessageHandlers = { __proto__: null };
+
+    constructor(suites, client) {
+        super(suites, client);
+        globalThis.addEventListener("message", this.onMessage.bind(this));
+    }
+
+    onMessage(event) {
+        const { cmd, data = undefined } = event.data;
+        const resolver = this._postMessageHandlers[cmd];
+        if (!resolver)
+            throw Error(`Unknown onMessage: ${cmd}`);
+        if (resolver === DONE)
+            throw Error(`Cannot evaluate ping back resolver twice: ${cmd}`);
+        this._postMessageHandlers[cmd] = DONE;
+        resolver(data);
+    }
+
+    async pingBackMessage(name) {
+        if (this._postMessageHandlers[name] !== DONE)
+            throw Error(`Unknown ping back message: ${name}`);
+        return new Promise(resolver => {
+            this._postMessageHandlers[name] = resolver;
+        });
+    }
+}
+
+export class SubdomainBenchmarkRunner extends PostMessageRunner {
+
+    _finishPromise;
+    _postMessageHandlers = {
+        __proto__: null,
+        prepareDone: DONE,
+        recordPrepareMetric: DONE,
+        recordTestResults: DONE,
+    };
+
+    // Assume all workloads report the metrics back via postMessage.
+    async _runSuite(suite) {
+        this._prepareMetrics(suite);
+        await this._prepareSuite(suite);
+        if (params.measurePrepare) {
+            const { prepareTime } = await this.pingBackMessage("recordPrepareMetric");
+            this._recordPrepareMetric(suite, prepareTime);
+        }
+        for (const test of suite.tests)
+            await this._runTestAndRecordResults(suite, test);
+    }
+
+    async _runTestAndRecordResults(suite, test) {
+        this.postMessage("runTestAndRecordResults");
+        const { syncTime, asyncTime } = await this.pingBackMessage("recordTestResults");
+        await this._recordTestResults(suite, test, syncTime, asyncTime);
+    }
+
+    postMessage(cmd, data) {
+        this._frame.contentWindow.postMessage({ cmd, data }, "*");
+    }
+
+    async _prepareSuite(suite) {
+        await new Promise((resolve) => {
+            const frame = this._page._frame;
+            frame.onload = async () => resolve();
+            const url = new URL(window.location.href);
+            const customParams = params.copy();
+            customParams.embedded = true;
+            customParams.suites = [suite.name];
+            customParams.tags = [];
+            customParams.startAutomatically = true;
+            customParams.developerMode = false;
+            customParams.iterationCount = 1;
+            // TODO: handle local test URLs better
+            if (this._isLocalUrl(url)) {
+                url.port = parseInt(url.port) + this._currentIteration;
+            } else {
+                // Input:  foo-bar.custom.domain.com,   foo-bar-0.custom.domain.com
+                // Output: foo-bar-1.custom.domain.com, foo-bar-1.custom.domain.com
+                const hostParts = url.hostname.split(".");
+                const domainWithoutIndex = hostParts[0].match(DOMAIN_WITHOUT_INDEX_RE).groups.nonIndexed;
+                hostParts[0] = `${domainWithoutIndex}-${this.currentIteration}`;
+                url.host = hostParts.join(".");
+            }
+            url.search = customParams.toSearchParams();
+            frame.src = url.toString();
+        });
+        await this.pingBackMessage("prepareDone");
+        this.postMessage("startSuite");
+    }
+
+    _isLocalUrl(url) {
+        return url.hostname === "127.0.0.1" || url.hostname === "localhost";
+    }
+}
+
+const DOMAIN_WITHOUT_INDEX_RE = /(?<nonIndexed>([^-]+(-[^0-9])?)+)(-[0-9]+)?/;
+
+export class EmbeddedBenchmarkRunner extends PostMessageRunner {
+
+    _postMessageHandlers = {
+        __proto__: null,
+        startSuite: DONE,
+        runTestAndRecordResults: DONE,
+    };
+
+    constructor(suites, client) {
+        super(suites, client);
+        if (globalThis === globalThis.parent) {
+            const msg = "Cannot use embedded runner in toplevel iframe";
+            alert(msg);
+            throw Error(msg);
+        }
+    }
+
+    postMessage(cmd, data) {
+        globalThis.parent.postMessage({ cmd, data }, "*");
+    }
+
+    async _prepareSuite(suite) {
+        await super._prepareSuite(suite);
+        this.postMessage("prepareDone", { prepareTime: 0 });
+        await this.pingBackMessage("startSuite");
+    }
+
+    async _runTestAndRecordResults(suite, test) {
+        await this.pingBackMessage("runTestAndRecordResults");
+        return super._runTestAndRecordResults(suite, test);
+    }
+
+    _recordPrepareMetric(suite, prepareTime) {
+        super._recordPrepareMetric(suite, prepareTime);
+        this.postMessage("recordPrepareMetric", { prepareTime });
+    }
+
+    async _recordTestResults(suite, test, syncTime, asyncTime) {
+        this.postMessage("recordTestResults", { syncTime, asyncTime });
+    }
+}

resources/developer-mode.mjs

@@ -23,8 +23,10 @@ export function createDeveloperModeContainer() {
     settings.append(createUIForWarmupBeforeSync());
     settings.append(createUIForSyncStepDelay());
     settings.append(createUIForLeakyIframes());
+    settings.append(createUIForMeasurePrepare());
+    settings.append(createUIForDomainPerIteration());
+    settings.append(createUIForForceGC());
 
-    content.append(document.createElement("hr"));
     content.append(settings);
 
     content.append(document.createElement("hr"));
@@ -92,6 +94,57 @@ function createUIForWarmupSuite() {
     return label;
 }
 
+function createUIForMeasurePrepare() {
+    let check = document.createElement("input");
+    check.type = "checkbox";
+    check.id = "measure-prepare";
+    check.checked = params.measurePrepare;
+
+    check.onchange = () => {
+        params.measurePrepare = check.checked;
+        updateURL();
+    };
+
+    let label = document.createElement("label");
+    label.append(check, " ", "Measure Prepare");
+
+    return label;
+}
+
+function createUIForDomainPerIteration() {
+    let check = document.createElement("input");
+    check.type = "checkbox";
+    check.id = "domain-per-iteration";
+    check.checked = params.domainPerIteration;
+
+    check.onchange = () => {
+        params.domainPerIteration = check.checked;
+        updateURL();
+    };
+
+    let label = document.createElement("label");
+    label.append(check, " ", "Use Subdomain-Runner");
+
+    return label;
+}
+
+function createUIForForceGC() {
+    let check = document.createElement("input");
+    check.type = "checkbox";
+    check.id = "suite-force-gc";
+    check.checked = params.suiteForceGC;
+
+    check.onchange = () => {
+        params.suiteForceGC = check.checked;
+        updateURL();
+    };
+
+    let label = document.createElement("label");
+    label.append(check, " ", "Force GC before every Suite");
+
+    return label;
+}
+
 function createUIForIterationCount() {
     const { range, label } = createTimeRangeUI("Iterations: ", params.iterationCount, "#", 1, 200);
     range.onchange = () => {
@@ -291,7 +344,7 @@ function updateURL() {
     else
         url.searchParams.delete("measurementMethod");
 
-    const boolParamKeys = ["iterationCount", "useWarmupSuite", "warmupBeforeSync", "waitBeforeSync", "leakyIframes"];
+    const boolParamKeys = ["iterationCount", "useWarmupSuite", "warmupBeforeSync", "waitBeforeSync", "domainPerIteration", "suiteForceGC", "leakyIframes"];
     for (const paramKey of boolParamKeys) {
         if (params[paramKey] !== defaultParams[paramKey])
             url.searchParams.set(paramKey, params[paramKey]);
@@ -299,6 +352,11 @@ function updateURL() {
             url.searchParams.delete(paramKey);
     }
 
+    if (params.measurePrepare !== defaultParams.measurePrepare)
+        url.searchParams.set("measurePrepare", params.measurePrepare);
+    else
+        url.searchParams.delete("measurePrepare");
+
     // Only push state if changed
     url.search = decodeURIComponent(url.search);
     if (url.href !== window.location.href)

resources/main.css

@@ -836,3 +836,21 @@ section#about .note {
     color: #fff;
     stroke: #fff;
 }
+
+
+.embedded section {
+    border: none;
+    position: absolute;
+    top: 0;
+    left: 0;
+    margin: 0;
+    padding: 0;
+}
+
+.embedded #logo,
+.embedded #progress,
+.embedded #info,
+.embedded #info-progress,
+.embedded #info-label {
+    display: none !important;
+}

resources/main.mjs

@@ -1,4 +1,4 @@
-import { BenchmarkRunner } from "./benchmark-runner.mjs";
+import { BenchmarkRunner, SubdomainBenchmarkRunner, EmbeddedBenchmarkRunner } from "./benchmark-runner.mjs";
 import * as Statistics from "./statistics.mjs";
 import { Suites } from "./tests.mjs";
 import { renderMetricView } from "./metric-ui.mjs";
@@ -12,11 +12,13 @@ class MainBenchmarkClient {
     suitesCount = null;
     _measuredValuesList = [];
     _finishedTestCount = 0;
+    _totalSubtestsCount = 0;
     _progressCompleted = null;
     _isRunning = false;
     _hasResults = false;
     _developerModeContainer = null;
     _metrics = Object.create(null);
+    _runner = null;
 
     constructor() {
         window.addEventListener("DOMContentLoaded", () => this.prepareUI());
@@ -60,14 +62,19 @@ class MainBenchmarkClient {
         this._isRunning = true;
 
         const enabledSuites = Suites.filter((suite) => !suite.disabled);
-        const totalSubtestsCount = enabledSuites.reduce((testsCount, suite) => {
+        this._totalSubtestsCount = enabledSuites.reduce((testsCount, suite) => {
             return testsCount + suite.tests.length;
         }, 0);
-        this.stepCount = params.iterationCount * totalSubtestsCount;
+        this.stepCount = params.iterationCount * this._totalSubtestsCount;
         this._progressCompleted.max = this.stepCount;
         this.suitesCount = enabledSuites.length;
-        const runner = new BenchmarkRunner(Suites, this);
-        runner.runMultipleIterations(params.iterationCount);
+        if (params.embedded)
+            this._runner = new EmbeddedBenchmarkRunner(Suites, this);
+        else if (params.domainPerIteration)
+            this._runner = new SubdomainBenchmarkRunner(Suites, this);
+        else
+            this._runner = new BenchmarkRunner(Suites, this);
+        this._runner.runMultipleIterations(params.iterationCount);
         return true;
     }
 
@@ -76,9 +83,6 @@ class MainBenchmarkClient {
     }
 
     willAddTestFrame(frame) {
-        frame.style.left = "50%";
-        frame.style.top = "50%";
-        frame.style.transform = "translate(-50%, -50%)";
     }
 
     willRunTest(suite, test) {
@@ -97,7 +101,7 @@ class MainBenchmarkClient {
 
     willStartFirstIteration() {
         this._measuredValuesList = [];
-        this._finishedTestCount = 0;
+        this._finishedTestCount = params.currentIteration * this._totalSubtestsCount;
     }
 
     didFinishLastIteration(metrics) {
@@ -105,6 +109,9 @@ class MainBenchmarkClient {
         this._isRunning = false;
         this._hasResults = true;
         this._metrics = metrics;
+        // Don't display the results if the runnier is embedded.
+        if (params.embedded)
+            return;
 
         const scoreResults = this._computeResults(this._measuredValuesList, "score");
         if (scoreResults.isValid)
@@ -213,8 +220,9 @@ class MainBenchmarkClient {
         const trackHeight = 24;
         document.documentElement.style.setProperty("--metrics-line-height", `${trackHeight}px`);
         const plotWidth = (params.viewport.width - 120) / 2;
-        document.getElementById("geomean-chart").innerHTML = renderMetricView({
-            metrics: [metrics.Geomean],
+        const singleToplevelMetrics = Object.values(metrics).filter((each) => each && each.unit === "ms" && !each.parent && !each.children.length);
+        document.getElementById("toplevel-chart").innerHTML = renderMetricView({
+            metrics: singleToplevelMetrics,
             width: plotWidth,
             trackHeight,
             renderChildren: false,
@@ -258,6 +266,8 @@ class MainBenchmarkClient {
     }
 
     prepareUI() {
+        if (params.embedded)
+            document.body.className = "embedded";
         window.addEventListener("hashchange", this._hashChangeHandler.bind(this));
         window.addEventListener("resize", this._resizeScreeHandler.bind(this));
         this._resizeScreeHandler();

resources/metric.mjs

@@ -5,7 +5,7 @@ export const MILLISECONDS_PER_MINUTE = 60 * 1000;
 export class Metric {
     static separator = "/";
 
-    constructor(name, unit = "ms") {
+    constructor(name, unit = "ms", values = []) {
         if (typeof name !== "string")
             throw new Error(`Invalid metric.name=${name}, expected string.`);
         this.name = name;
@@ -26,11 +26,17 @@ export class Metric {
         this.min = 0.0;
         this.max = 0.0;
 
-        this.values = [];
+        this.values = values;
 
         // Mark properties which refer to other Metric objects as
         // non-enumerable to avoid issue with JSON.stringify due to circular
         // references.
+        Object.defineProperties(this, {
+            children: {
+                writable: true,
+                value: [],
+            },
+        });
         Object.defineProperties(this, {
             parent: {
                 writable: true,

resources/params.mjs

@@ -1,4 +1,4 @@
-class Params {
+export class Params {
     viewport = {
         width: 800,
         height: 600,
@@ -27,11 +27,19 @@ class Params {
     shuffleSeed = "off";
     // Leak iframes to DevTools can display sources.
     leakyIframes = false;
+    // Emit metrics for the prepare phase.
+    measurePrepare = false;
+    // Use a new subdomain for each iteration to avoid caching.
+    domainPerIteration = false;
+    // Force a gc before every suite (chrome specific)
+    suiteForceGC = false;
+    currentIteration = 0;
+    embedded = false;
 
-    constructor(searchParams = undefined) {
+    constructor(searchParams = undefined, freeze = true) {
         if (searchParams)
             this._copyFromSearchParams(searchParams);
-        if (!this.developerMode) {
+        if (!this.developerMode && freeze) {
             Object.freeze(this.viewport);
             Object.freeze(this);
         }
@@ -65,24 +73,44 @@ class Params {
         if (searchParams.has("suite") || searchParams.has("suites")) {
             if (searchParams.has("suite") && searchParams.has("suites"))
                 throw new Error("Params 'suite' and 'suites' can not be used together.");
-            const value = searchParams.get("suite") || searchParams.get("suites");
-            this.suites = value.split(",");
-            if (this.suites.length === 0)
-                throw new Error("No suites selected");
+            const value = (searchParams.get("suite") || searchParams.get("suites")).trim();
+            if (value.length ) {
+                this.suites = value.split(",");
+                if (this.suites.length === 0)
+                    throw new Error("No suites selected");
+            }
             searchParams.delete("suite");
             searchParams.delete("suites");
         }
 
         if (searchParams.has("tags")) {
-            if (this.suites.length)
-                throw new Error("'suites' and 'tags' cannot be used together.");
-            this.tags = searchParams.get("tags").split(",");
+            const rawTags = searchParams.get("tags");
+            if (rawTags) {
+                this.tags = rawTags.split(",");
+                if (this.tags.length && this.suites.length)
+                    throw new Error("'suites' and 'tags' cannot be used together.");
+            }
             searchParams.delete("tags");
         }
 
+        this.embedded = searchParams.has("embedded");
+        searchParams.delete("embedded");
+
         this.developerMode = searchParams.has("developerMode");
         searchParams.delete("developerMode");
 
+        this.measurePrepare = searchParams.has("measurePrepare");
+        searchParams.delete("measurePrepare");
+
+        this.domainPerIteration = searchParams.has("domainPerIteration");
+        searchParams.delete("domainPerIteration");
+
+        this.suiteForceGC = searchParams.has("suiteForceGC");
+        searchParams.delete("suiteForceGC");
+
+        this.currentIteration = searchParams.get("currentIteration") | 0;
+        searchParams.delete("currentIteration");
+
         if (searchParams.has("useWarmupSuite")) {
             this.useWarmupSuite = true;
             searchParams.delete("useWarmupSuite");
@@ -134,10 +162,26 @@ class Params {
             console.error("Got unused search params", unused);
     }
 
+    toSearchParamsObject() {
+        const rawParams = { __proto__: null };
+        for (const [key, value] of Object.entries(this)) {
+            if (value === defaultParams[key])
+                continue;
+            rawParams[key] = value;
+        }
+        // Either suites or params can be used at the same time.
+        if (rawParams.suites?.length && rawParams.tags?.length)
+            delete rawParams.suites;
+        rawParams.viewport = `${this.viewport.width}x${this.viewport.height}`;
+        return new URLSearchParams(rawParams);
+    }
+
     toSearchParams() {
-        const rawParams = { ...this };
-        rawParams["viewport"] = `${this.viewport.width}x${this.viewport.height}`;
-        return new URLSearchParams(rawParams).toString();
+        return this.toSearchParamsObject().toString();
+    }
+
+    copy() {
+        return new Params(this.toSearchParamsObject(), false);
     }
 }
 
@@ -152,3 +196,4 @@ try {
     alert(`Invalid URL Param: ${e}`);
 }
 export const params = maybeCustomParams;
+globalThis.params = maybeCustomParams;

resources/tests.mjs

@@ -304,6 +304,39 @@ Suites.push({
     ],
 });
 
+// Downstream only.
+Suites.push({
+    name: "TodoMVC-React-Window",
+    url: "todomvc/architecture-examples/react/dist/index-window.html#/home",
+    tags: ["todomvc"],
+    disabled: true,
+    async prepare(page) {
+        await page.targetChildWindow();
+        const element = await page.waitForElement(".new-todo");
+        element.focus();
+    },
+    tests: [
+        new BenchmarkTestStep(`Adding${numberOfItemsToAdd}Items`, (page) => {
+            const newTodo = page.querySelector(".new-todo");
+            for (let i = 0; i < numberOfItemsToAdd; i++) {
+                newTodo.setValue(getTodoText(defaultLanguage, i));
+                newTodo.dispatchEvent("input");
+                newTodo.enter("keydown");
+            }
+        }),
+        new BenchmarkTestStep("CompletingAllItems", (page) => {
+            const checkboxes = page.querySelectorAll(".toggle");
+            for (let i = 0; i < numberOfItemsToAdd; i++)
+                checkboxes[i].click();
+        }),
+        new BenchmarkTestStep("DeletingAllItems", (page) => {
+            const deleteButtons = page.querySelectorAll(".destroy");
+            for (let i = numberOfItemsToAdd - 1; i >= 0; i--)
+                deleteButtons[i].click();
+        }),
+    ],
+});
+
 Suites.push({
     name: "TodoMVC-React-Redux",
     url: "todomvc/architecture-examples/react-redux/dist/index.html",
@@ -1036,6 +1069,7 @@ Suites.push({
     ],
 });
 
+
 Object.freeze(Suites);
 Suites.forEach((suite) => {
     if (!suite.tags)

resources/todomvc/architecture-examples/react/README.md

@@ -6,6 +6,8 @@ This application uses React 17.0.2 to implement a todo application.
 
 -   [React](https://reactjs.org/) is a JavaScript library for creating user interfaces.
 
+The downstream version also generates the TodoMVC-React-Window workload. A variation of TodoMVC-React that renders and controls the app in a new window using the scripts loaded in the parent window.
+
 ## Implementation details
 
 React:\
@@ -22,6 +24,8 @@ Controller: serves as an interface between view & model components
 
 To build the static files, this application utilizes webpack. It minifies and optimizes output files and copies all necessary files to a `dist` folder.
 
+For TodoMVC-React-Window, the webpack scripts separate the JavaScript from the CSS. The JavaScript is loaded by index.html in the main window, while the css is loaded by todo.html in the child window.
+
 ## Requirements
 
 The only requirement is an installation of Node, to be able to install dependencies and run scripts to serve a local server.

resources/todomvc/architecture-examples/react/dist/index-window.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en" data-framework="react" data-version="17.0.2" data-features=""><head><meta charset="UTF-8"/><meta name="description" content="A TodoMVC workload app for Speedometer!"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta http-equiv="X-UA-Compatible" content="ie=edge"/><title>TodoMVC: React-Window</title><script defer="defer" src="windowed.bundle.js"></script></head></html>

resources/todomvc/architecture-examples/react/dist/todo.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en" data-framework="react" data-version="17.0.2" data-features=""><head><meta charset="UTF-8"/><meta name="description" content="A TodoMVC workload app for Speedometer!"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta http-equiv="X-UA-Compatible" content="ie=edge"/><link href="windowed.css" rel="stylesheet"><title>TodoMVC: React-Window inner window</title></head><body><section class="todoapp" id="root"></section><footer class="info"><p>Click on input field to write your todo.</p><p>At least two characters are needed to be a valid entry.</p><p>Press 'enter' to add the todo.</p><p>Double-click to edit a todo</p></footer></body></html>

resources/todomvc/architecture-examples/react/dist/windowed.bundle.js.LICENSE.txt

@@ -0,0 +1,69 @@
+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+
+/*!
+	Copyright (c) 2018 Jed Watson.
+	Licensed under the MIT License (MIT), see
+	http://jedwatson.github.io/classnames
+*/
+
+/**
+ * @remix-run/router v1.3.3
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */
+
+/**
+ * React Router v6.8.2
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */
+
+/** @license React v0.20.2
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

resources/todomvc/architecture-examples/react/public/window/index.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en" data-framework="react" data-version="17.0.2" data-features="">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="description" content="A TodoMVC workload app for Speedometer!" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+        <title>TodoMVC: React-Window</title>
+        <script defer="defer" src=<%= htmlWebpackPlugin.options.script %> ></script>
+    </head>
+</html>

resources/todomvc/architecture-examples/react/public/window/todo.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html lang="en" data-framework="react" data-version="17.0.2" data-features="">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="description" content="A TodoMVC workload app for Speedometer!" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+        <link href=<%= htmlWebpackPlugin.options.styles %> rel="stylesheet">
+        <title>TodoMVC: React-Window inner window</title>
+    </head>
+    <body>
+        <section class="todoapp" id="root"></section>
+        <footer class="info">
+            <p>Click on input field to write your todo.</p>
+            <p>At least two characters are needed to be a valid entry.</p>
+            <p>Press 'enter' to add the todo.</p>
+            <p>Double-click to edit a todo</p>
+        </footer>
+    </body>
+</html>

resources/todomvc/architecture-examples/react/src/index-window.js

@@ -0,0 +1,26 @@
+import React from "react";
+import { render } from "react-dom";
+import { HashRouter, Route, Routes } from "react-router-dom";
+
+import { App } from "./todo/app";
+import "todomvc-app-css/index.css";
+
+let childWindow = window.open("todo.html", "_blank", "width=800,height=600");
+
+let AsyncLoop = () => {
+    if (childWindow.document.getElementById("root")) {
+        render(
+            <HashRouter>
+                <Routes>
+                    <Route path="*" element={<App />} />
+                </Routes>
+            </HashRouter>,
+            childWindow.document.getElementById("root")
+        );
+        globalThis.childWindow = childWindow;
+    } else {
+        setTimeout(AsyncLoop, 10);
+    }
+};
+
+AsyncLoop();

resources/todomvc/architecture-examples/react/webpack.common.js

@@ -4,12 +4,28 @@ const path = require("path");
 module.exports = {
     entry: {
         app: path.resolve(__dirname, "src", "index.js"),
+        windowed: path.resolve(__dirname, "src", "index-window.js"),
     },
     plugins: [
         new HtmlWebpackPlugin({
             title: "TodoMVC: React",
+            chunks: ["app"],
             template: path.resolve(__dirname, "public", "index.html"),
         }),
+        new HtmlWebpackPlugin({
+            title: "TodoMVC: React-Window",
+            inject: false,
+            template: path.resolve(__dirname, "public", "window", "index.html"),
+            filename: "index-window.html",
+            script: "windowed.bundle.js",
+        }),
+        new HtmlWebpackPlugin({
+            title: "TodoMVC: React-Window inner window",
+            inject: false,
+            template: path.resolve(__dirname, "public", "window", "todo.html"),
+            filename: "todo.html",
+            styles: "windowed.css",
+        }),
     ],
     output: {
         filename: "[name].bundle.js",