Remote Runner / PostMessge api (#456)

resources/benchmark-runner.mjs

@@ -1,5 +1,5 @@
 import { Metric } from "./metric.mjs";
-import { params } from "./params.mjs";
+import { params } from "./shared/params.mjs";
 import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs";
 
 const performance = globalThis.performance;

resources/developer-mode.mjs

@@ -1,5 +1,5 @@
 import { Suites, Tags } from "./tests.mjs";
-import { params, defaultParams } from "./params.mjs";
+import { params, defaultParams } from "./shared/params.mjs";
 
 export function createDeveloperModeContainer() {
     const container = document.createElement("div");

resources/main.mjs

@@ -2,7 +2,7 @@ import { BenchmarkRunner } from "./benchmark-runner.mjs";
 import * as Statistics from "./statistics.mjs";
 import { Suites } from "./tests.mjs";
 import { renderMetricView } from "./metric-ui.mjs";
-import { params } from "./params.mjs";
+import { params } from "./shared/params.mjs";
 import { createDeveloperModeContainer } from "./developer-mode.mjs";
 
 // FIXME(camillobruni): Add base class

resources/newssite/news-next/dist/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/pages/_error-54de1933a164a1ff.js" defer=""></script><script src="./_next/static/ofW8d8vHz4HS9u6cDJv4X/_buildManifest.js" defer=""></script><script src="./_next/static/ofW8d8vHz4HS9u6cDJv4X/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"ofW8d8vHz4HS9u6cDJv4X","assetPrefix":".","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/pages/_error-54de1933a164a1ff.js" defer=""></script><script src="./_next/static/tdun_naEsLSWD7CGin1GQ/_buildManifest.js" defer=""></script><script src="./_next/static/tdun_naEsLSWD7CGin1GQ/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"tdun_naEsLSWD7CGin1GQ","assetPrefix":".","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

resources/newssite/news-next/dist/_next/static/tdun_naEsLSWD7CGin1GQ/_buildManifest.js

@@ -1 +1 @@
-self.__BUILD_MANIFEST={__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":["static/chunks/743-fd706aeabb7828e3.js","static/css/2cf5163b53bb0adb.css","static/chunks/pages/index-7fd4a7c8e35958df.js"],"/_error":["static/chunks/pages/_error-54de1933a164a1ff.js"],sortedPages:["/","/_app","/_error"]},self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();
+self.__BUILD_MANIFEST={__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":["static/chunks/743-fd706aeabb7828e3.js","static/css/2cf5163b53bb0adb.css","static/chunks/pages/index-ca407dccff56c060.js"],"/_error":["static/chunks/pages/_error-54de1933a164a1ff.js"],sortedPages:["/","/_app","/_error"]},self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();

resources/newssite/news-next/dist/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><link rel="preload" href="./_next/static/css/2cf5163b53bb0adb.css" as="style"/><link rel="stylesheet" href="./_next/static/css/2cf5163b53bb0adb.css" data-n-p=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/743-fd706aeabb7828e3.js" defer=""></script><script src="./_next/static/chunks/pages/index-7fd4a7c8e35958df.js" defer=""></script><script src="./_next/static/ofW8d8vHz4HS9u6cDJv4X/_buildManifest.js" defer=""></script><script src="./_next/static/ofW8d8vHz4HS9u6cDJv4X/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"ofW8d8vHz4HS9u6cDJv4X","assetPrefix":".","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><link rel="preload" href="./_next/static/css/2cf5163b53bb0adb.css" as="style"/><link rel="stylesheet" href="./_next/static/css/2cf5163b53bb0adb.css" data-n-p=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/743-fd706aeabb7828e3.js" defer=""></script><script src="./_next/static/chunks/pages/index-ca407dccff56c060.js" defer=""></script><script src="./_next/static/tdun_naEsLSWD7CGin1GQ/_buildManifest.js" defer=""></script><script src="./_next/static/tdun_naEsLSWD7CGin1GQ/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"tdun_naEsLSWD7CGin1GQ","assetPrefix":".","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>

resources/newssite/news-next/package-lock.json

@@ -18,9 +18,20 @@
                 "react-dom": "18.2.0",
                 "react-router-dom": "^6.11.1",
                 "react-router-hash-link": "^2.4.3",
+                "speedometer-utils": "../../shared",
                 "uuid": "^9.0.0"
             }
         },
+        "../../shared": {
+            "name": "speedometer-utils",
+            "version": "1.0.0"
+        },
+        "../../test-invoker.mjs": {
+            "extraneous": true
+        },
+        "../../test-runner.mjs": {
+            "extraneous": true
+        },
         "../news-site-css": {
             "version": "1.8.3",
             "license": "ISC",
@@ -3467,6 +3478,10 @@
                 "node": ">=0.10.0"
             }
         },
+        "node_modules/speedometer-utils": {
+            "resolved": "../../shared",
+            "link": true
+        },
         "node_modules/stop-iteration-iterator": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz",
@@ -6241,6 +6256,9 @@
             "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz",
             "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw=="
         },
+        "speedometer-utils": {
+            "version": "file:../../shared"
+        },
         "stop-iteration-iterator": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz",

resources/newssite/news-next/package.json

@@ -20,6 +20,7 @@
         "react-dom": "18.2.0",
         "react-router-dom": "^6.11.1",
         "react-router-hash-link": "^2.4.3",
+        "speedometer-utils": "../../shared",
         "uuid": "^9.0.0"
     }
 }

resources/newssite/news-next/src/pages/index.js

@@ -1,10 +1,21 @@
-import React from "react";
+import { useEffect } from "react";
 import { HashRouter as Router, Routes, Route } from "react-router-dom";
 import Page from "@/partials/page/page";
 import Head from "next/head";
 import { DataContextProvider } from "@/context/data-context";
+import { BenchmarkConnector } from "speedometer-utils/workload-testing-utils.mjs";
+import suites from "@/workload-test.mjs";
 
 export default function App() {
+    // Using 'useLayoutEffect' here, since this will connect the workload after all DOM mutations happened.
+    // This ensures that all elemetns are in the DOM, prior to signaling to the Benchmark that the workload is ready to run a test suite.
+    useEffect(() => {
+        const benchmarkConnector = new BenchmarkConnector(suites, "news-next", 1);
+        benchmarkConnector.connect();
+
+        return () => benchmarkConnector.disconnect();
+    }, []);
+
     return (
         <>
             <Head>

resources/newssite/news-next/src/workload-test.mjs

@@ -0,0 +1,42 @@
+import { BenchmarkStep, BenchmarkSuite } from "speedometer-utils/benchmark.mjs";
+import { forceLayout, getElement } from "speedometer-utils/helpers.mjs";
+
+const suites = {
+    default: new BenchmarkSuite("default", [
+        new BenchmarkStep("Navigate-to-US-page", () => {
+            for (let i = 0; i < 25; i++) {
+                getElement("#navbar-dropdown-toggle").click();
+                forceLayout();
+                getElement("#navbar-dropdown-toggle").click();
+                forceLayout();
+            }
+
+            getElement("#navbar-navlist-us-link").click();
+            forceLayout();
+        }),
+        new BenchmarkStep("Navigate-to-World-page", () => {
+            for (let i = 0; i < 25; i++) {
+                getElement("#navbar-dropdown-toggle").click();
+                forceLayout();
+                getElement("#navbar-dropdown-toggle").click();
+                forceLayout();
+            }
+
+            getElement("#navbar-navlist-world-link").click();
+            forceLayout();
+        }),
+        new BenchmarkStep("Navigate-to-Politics-page", () => {
+            for (let i = 0; i < 25; i++) {
+                getElement("#navbar-dropdown-toggle").click();
+                forceLayout();
+                getElement("#navbar-dropdown-toggle").click();
+                forceLayout();
+            }
+
+            getElement("#navbar-navlist-politics-link").click();
+            forceLayout();
+        }),
+    ]),
+};
+
+export default suites;

resources/shared/benchmark.mjs

@@ -0,0 +1,127 @@
+/* eslint-disable no-case-declarations */
+import { TestRunner } from "./test-runner.mjs";
+import { Params } from "./params.mjs";
+
+/**
+ * BenchmarkStep
+ *
+ * A single test step, with a common interface to interact with.
+ */
+export class BenchmarkStep {
+    constructor(name, run) {
+        this.name = name;
+        this.run = run;
+    }
+
+    async runAndRecord(params, suite, test, callback) {
+        const testRunner = new TestRunner(null, null, params, suite, test, callback);
+        const result = await testRunner.runTest();
+        return result;
+    }
+}
+
+/**
+ * BenchmarkSuite
+ *
+ * A single test suite that contains one or more test steps.
+ */
+export class BenchmarkSuite {
+    constructor(name, tests) {
+        this.name = name;
+        this.tests = tests;
+    }
+
+    record(_test, syncTime, asyncTime) {
+        const total = syncTime + asyncTime;
+        const results = {
+            tests: { Sync: syncTime, Async: asyncTime },
+            total: total,
+        };
+
+        return results;
+    }
+
+    async runAndRecord(params, onProgress) {
+        const measuredValues = {
+            tests: {},
+            total: 0,
+        };
+        const suiteStartLabel = `suite-${this.name}-start`;
+        const suiteEndLabel = `suite-${this.name}-end`;
+
+        performance.mark(suiteStartLabel);
+
+        for (const test of this.tests) {
+            const result = await test.runAndRecord(params, this, test, this.record);
+            measuredValues.tests[test.name] = result;
+            measuredValues.total += result.total;
+            onProgress?.(test.name);
+        }
+
+        performance.mark(suiteEndLabel);
+        performance.measure(`suite-${this.name}`, suiteStartLabel, suiteEndLabel);
+
+        return {
+            type: "suite-tests-complete",
+            status: "success",
+            result: measuredValues,
+            suitename: this.name,
+        };
+    }
+}
+
+/** **********************************************************************
+ * BenchmarkConnector
+ *
+ * postMessage is used to communicate between app and benchmark.
+ * When the app is ready, an 'app-ready' message is sent to signal that the app can receive instructions.
+ *
+ * A prepare script within the apps appends window.name and window.version from the package.json file.
+ * The appId is build by appending name-version
+ * It's used as an additional safe-guard to ensure the correct app responds to a message.
+ *************************************************************************/
+export class BenchmarkConnector {
+    constructor(suites, name, version) {
+        this.suites = suites;
+        this.name = name;
+        this.version = version;
+
+        if (!name || !version)
+            console.warn("No name or version supplied, to create a unique appId");
+
+        this.appId = name && version ? `${name}-${version}` : -1;
+        this.onMessage = this.onMessage.bind(this);
+    }
+
+    async onMessage(event) {
+        if (event.data.id !== this.appId || event.data.key !== "benchmark-connector")
+            return;
+
+        switch (event.data.type) {
+            case "benchmark-suite":
+                const params = new Params(new URLSearchParams(window.location.search));
+                const suite = this.suites[event.data.name];
+                if (!suite)
+                    console.error(`Suite with the name of "${event.data.name}" not found!`);
+                const { result } = await suite.runAndRecord(params, (test) => this.sendMessage({ type: "step-complete", status: "success", appId: this.appId, name: this.name, test }));
+                this.sendMessage({ type: "suite-complete", status: "success", appId: this.appId, result });
+                this.disconnect();
+                break;
+            default:
+                console.error(`Message data type not supported: ${event.data.type}`);
+        }
+    }
+
+    sendMessage(message) {
+        window.top.postMessage(message, "*");
+    }
+
+    connect() {
+        window.addEventListener("message", this.onMessage);
+        this.sendMessage({ type: "app-ready", status: "success", appId: this.appId });
+    }
+
+    disconnect() {
+        window.removeEventListener("message", this.onMessage);
+    }
+}

resources/shared/helpers.mjs

@@ -0,0 +1,30 @@
+/**
+ * Helper Methods
+ *
+ * Various methods that are extracted from the Page class.
+ */
+export function getParent(lookupStartNode, path) {
+    lookupStartNode = lookupStartNode.shadowRoot ?? lookupStartNode;
+    const parent = path.reduce((root, selector) => {
+        const node = root.querySelector(selector);
+        return node.shadowRoot ?? node;
+    }, lookupStartNode);
+
+    return parent;
+}
+
+export function getElement(selector, path = [], lookupStartNode = document) {
+    const element = getParent(lookupStartNode, path).querySelector(selector);
+    return element;
+}
+
+export function getAllElements(selector, path = [], lookupStartNode = document) {
+    const elements = Array.from(getParent(lookupStartNode, path).querySelectorAll(selector));
+    return elements;
+}
+
+export function forceLayout() {
+    const rect = document.body.getBoundingClientRect();
+    const e = document.elementFromPoint((rect.width / 2) | 0, (rect.height / 2) | 0);
+    return e;
+}

resources/shared/package.json

@@ -0,0 +1,5 @@
+{
+    "name": "speedometer-utils",
+    "version": "1.0.0",
+    "description": "Utility files for Speedometer & Workloads"
+}

resources/shared/params.mjs

@@ -1,4 +1,4 @@
-class Params {
+export class Params {
     viewport = {
         width: 800,
         height: 600,
@@ -108,7 +108,7 @@ class Params {
         return defaultParams.suites;
     }
 
-    _parseTags() {
+    _parseTags(searchParams) {
         if (!searchParams.has("tags"))
             return defaultParams.tags;
         if (this.suites.length)
@@ -146,21 +146,34 @@ class Params {
         return shuffleSeed;
     }
 
+    toSearchParamsObject() {
+        const rawParams = { __proto__: null };
+        for (const [key, value] of Object.entries(this)) {
+            if (value === defaultParams[key])
+                continue;
+            rawParams[key] = value;
+        }
+
+        // Either suites or params can be used at the same time.
+        if (rawParams.suites?.length && rawParams.tags?.length)
+            delete rawParams.suites;
+        rawParams.viewport = `${this.viewport.width}x${this.viewport.height}`;
+
+        return new URLSearchParams(rawParams);
+    }
+
     toSearchParams() {
-        const rawParams = { ...this };
-        rawParams["viewport"] = `${this.viewport.width}x${this.viewport.height}`;
-        return new URLSearchParams(rawParams).toString();
+        return this.toSearchParamsObject().toString();
     }
 }
 
 export const defaultParams = new Params();
 
-const searchParams = new URLSearchParams(window.location.search);
+const searchParams = new URLSearchParams(typeof window !== "undefined" ? window.location.search : undefined);
 let maybeCustomParams = new Params();
 try {
     maybeCustomParams = new Params(searchParams);
 } catch (e) {
     console.error("Invalid URL Param", e, "\nUsing defaults as fallback:", maybeCustomParams);
-    alert(`Invalid URL Param: ${e}`);
 }
 export const params = maybeCustomParams;

resources/shared/test-invoker.mjs

@@ -15,8 +15,8 @@ export class TimerTestInvoker extends TestInvoker {
                 setTimeout(() => {
                     this._asyncCallback();
                     requestAnimationFrame(async () => {
-                        await this._reportCallback();
-                        resolve();
+                        const result = await this._reportCallback();
+                        resolve(result);
                     });
                 }, 0);
             }, this._params.waitBeforeSync);
@@ -40,8 +40,8 @@ export class RAFTestInvoker extends TestInvoker {
             setTimeout(() => {
                 this._asyncCallback();
                 setTimeout(async () => {
-                    await this._reportCallback();
-                    resolve();
+                    const result = await this._reportCallback();
+                    resolve(result);
                 }, 0);
             }, 0);
         });

resources/shared/workload-testing-utils.mjs

@@ -0,0 +1,2 @@
+export * from "./helpers.mjs";
+export * from "./benchmark.mjs";

resources/suite-runner.mjs

@@ -1,8 +1,6 @@
-import { TestRunner } from "./test-runner.mjs";
+import { TestRunner } from "./shared/test-runner.mjs";
 import { WarmupSuite } from "./benchmark-runner.mjs";
 
-// FIXME: Create AsyncSuiteRunner subclass.
-// FIXME: Create RemoteSuiteRunner subclass.
 export class SuiteRunner {
     #frame;
     #page;
@@ -25,6 +23,30 @@ export class SuiteRunner {
         this.#params = params;
     }
 
+    get frame() {
+        return this.#frame;
+    }
+
+    get page() {
+        return this.#page;
+    }
+
+    get params() {
+        return this.#params;
+    }
+
+    get suite() {
+        return this.#suite;
+    }
+
+    get client() {
+        return this.#client;
+    }
+
+    get suiteResults() {
+        return this.#suiteResults;
+    }
+
     async run() {
         await this._prepareSuite();
         await this._runSuite();
@@ -60,8 +82,7 @@ export class SuiteRunner {
 
         performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
         this._validateSuiteTotal();
-        if (this.#client?.didFinishSuite)
-            await this.#client.didFinishSuite(this.#suite);
+        await this._updateClient();
     }
 
     _validateSuiteTotal() {
@@ -78,7 +99,7 @@ export class SuiteRunner {
             const frame = this.#frame;
             frame.onload = () => resolve();
             frame.onerror = () => reject();
-            frame.src = this.#suite.url;
+            frame.src = `${this.#suite.url}?${this.#params.toSearchParams()}`;
         });
     }
 
@@ -90,14 +111,105 @@ export class SuiteRunner {
         const total = syncTime + asyncTime;
         this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
         this.#suiteResults.total += total;
-
-        if (this.#client?.didRunTest)
-            await this.#client.didRunTest(this.#suite, test);
     };
+
+    async _updateClient(suite = this.#suite) {
+        if (this.#client?.didFinishSuite)
+            await this.#client.didFinishSuite(suite);
+    }
 }
 
-// FIXME: implement remote steps
-class RemoteSuiteRunner extends SuiteRunner {}
+export class RemoteSuiteRunner extends SuiteRunner {
+    #appId;
+
+    get appId() {
+        return this.#appId;
+    }
+
+    set appId(id) {
+        this.#appId = id;
+    }
+
+    async run() {
+        this.postMessageCallbacks = new Map();
+        const handler = this._handlePostMessage.bind(this);
+        window.addEventListener("message", handler);
+
+        // FIXME: use this._suite in all SuiteRunner methods directly.
+        try {
+            await this._prepareSuite();
+            await this._runSuite();
+        } finally {
+            window.removeEventListener("message", handler);
+        }
+    }
+
+    async _prepareSuite() {
+        const suiteName = this.suite.name;
+        const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
+        const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
+
+        performance.mark(suitePrepareStartLabel);
+
+        // Wait for the app-ready message from the workload.
+        const appReadyPromise = this._subscribeOnce("app-ready");
+        await this._loadFrame(this.suite);
+        const response = await appReadyPromise;
+        await this.suite.prepare(this.page);
+        // Capture appId to pass along with messages.
+        this.appId = response?.appId;
+
+        performance.mark(suitePrepareEndLabel);
+
+        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+    }
+
+    async _runSuite() {
+        // Ask workload to run its own tests.
+        this.frame.contentWindow.postMessage({ id: this.appId, key: "benchmark-connector", type: "benchmark-suite", name: this.suite.config?.name || "default" }, "*");
+        // Capture metrics from the completed tests.
+        const response = await this._subscribeOnce("suite-complete");
+
+        this.suiteResults.tests = {
+            ...this.suiteResults.tests,
+            ...response.result.tests,
+        };
+
+        this.suiteResults.total += response.result.total;
+
+        this._validateSuiteTotal();
+        await this._updateClient();
+    }
+
+    _handlePostMessage(event) {
+        const callback = this.postMessageCallbacks.get(event.data.type);
+        if (callback)
+            callback(event);
+    }
+
+    _startSubscription(type, callback) {
+        if (this.postMessageCallbacks.has(type))
+            throw new Error("Callback exists already");
+
+        this.postMessageCallbacks.set(type, callback);
+    }
+
+    _stopSubscription(type) {
+        if (!this.postMessageCallbacks.has(type))
+            throw new Error("Callback does not exist");
+
+        this.postMessageCallbacks.delete(type);
+    }
+
+    _subscribeOnce(type) {
+        return new Promise((resolve) => {
+            this._startSubscription(type, (e) => {
+                this._stopSubscription(type);
+                resolve(e.data);
+            });
+        });
+    }
+}
 
 export const SUITE_RUNNER_LOOKUP = {
     __proto__: null,

resources/tests.mjs

@@ -855,7 +855,7 @@ Suites.push({
 
 Suites.push({
     name: "NewsSite-Next",
-    url: "resources/newssite/news-next/dist/index.html#/home",
+    url: "resources/newssite/news-next/dist/index.html",
     tags: ["newssite", "language"],
     async prepare(page) {
         await page.waitForElement("#navbar-dropdown-toggle");
@@ -894,6 +894,18 @@ Suites.push({
     ],
 });
 
+Suites.push({
+    name: "NewsSite-PostMessage",
+    url: "resources/newssite/news-next/dist/index.html",
+    tags: ["experimental", "newssite", "language"],
+    disabled: true,
+    async prepare() {},
+    type: "remote",
+    /* config: {
+        name: "default", // optional param to target non-default tests locally
+    }, */
+});
+
 Suites.push({
     name: "NewsSite-Nuxt",
     url: "resources/newssite/news-nuxt/dist/index.html",

tests/benchmark-runner-tests.mjs

@@ -1,7 +1,7 @@
 import { BenchmarkRunner } from "../resources/benchmark-runner.mjs";
 import { SuiteRunner } from "../resources/suite-runner.mjs";
-import { TestRunner } from "../resources/test-runner.mjs";
-import { defaultParams } from "../resources/params.mjs";
+import { TestRunner } from "../resources/shared/test-runner.mjs";
+import { defaultParams } from "../resources/shared/params.mjs";
 
 function TEST_FIXTURE(name) {
     return {
@@ -25,7 +25,7 @@ const SUITES_FIXTURE = [
 
 const CLIENT_FIXTURE = {
     willRunTest: sinon.stub(),
-    didRunTest: sinon.stub(),
+    didFinishSuite: sinon.stub(),
     didRunSuites: sinon.stub(),
 };
 
@@ -179,6 +179,7 @@ describe("BenchmarkRunner", () => {
                 assert.calledWith(performanceMarkSpy, "suite-Suite 1-start");
                 assert.calledWith(performanceMarkSpy, "suite-Suite 1-end");
                 expect(performanceMarkSpy.callCount).to.equal(4);
+                assert.calledOnce(runner._client.didFinishSuite);
             });
         });
     });
@@ -200,7 +201,6 @@ describe("BenchmarkRunner", () => {
 
             it("should run client pre and post hooks if present", () => {
                 assert.calledWith(runner._client.willRunTest, suite, suite.tests[0]);
-                assert.calledWith(runner._client.didRunTest, suite, suite.tests[0]);
             });
 
             it("should write performance marks at the start and end of the test with the correct test name", () => {