Don't display invalid results (#406) (#479)

This partially fixes #399 by not display non-finite scores which are caused by 0-measured suite results.

- For non-positive or non-finite scores, the following summary page is displayed. Note that the detailed page is still accessible since you might find some useful information there.
- Additionally, a console.error is generated for each zero-sum suite
- Merge of original PR  #406

index.html

@@ -50,7 +50,7 @@
                 </div>
             </section>
 
-            <section id="summary" data-title="Results Summary">
+            <section id="summary" data-title="Results Summary" class="valid">
                 <a href="#home" class="logo">
                     <img srcset="resources/logo@2x.png 2x" src="resources/logo.png" alt="Speedometer" />
                     <div class="version">3.1</div>
@@ -62,6 +62,10 @@
                 <hr />
                 <div id="result-number"></div>
                 <div id="confidence-number"></div>
+                <div id="invalid-score-text">
+                    One or more subtests produced no duration.<br />
+                    Please check your <a href="./instructions.html" target="_blank">browser settings</a> and re-run the benchmark.<br />
+                </div>
                 <div class="buttons">
                     <div class="button-row">
                         <a class="button" href="#details" id="show-details" title="Show detailed results data.">Details</a>

resources/benchmark-runner.mjs

@@ -339,17 +339,29 @@ export class BenchmarkRunner {
         if (this._client?.willStartFirstIteration)
             await this._client.willStartFirstIteration(iterationCount);
 
+        try {
+            await this._runMultipleIterations();
+        } catch (error) {
+            console.error(error);
+            if (this._client?.handleError) {
+                await this._client.handleError(error);
+                return;
+            }
+        }
+
+        if (this._client?.didFinishLastIteration)
+            await this._client.didFinishLastIteration(this._metrics);
+    }
+
+    async _runMultipleIterations() {
         const iterationStartLabel = "iteration-start";
         const iterationEndLabel = "iteration-end";
-        for (let i = 0; i < iterationCount; i++) {
+        for (let i = 0; i < this._iterationCount; i++) {
             performance.mark(iterationStartLabel);
             await this._runAllSuites();
             performance.mark(iterationEndLabel);
             performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel);
         }
-
-        if (this._client?.didFinishLastIteration)
-            await this._client.didFinishLastIteration(this._metrics);
     }
 
     _removeFrame() {
@@ -402,15 +414,21 @@ export class BenchmarkRunner {
                 suites[j] = tmp;
             }
         }
-
         performance.mark(prepareEndLabel);
         performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel);
 
-        for (const suite of suites) {
-            if (!suite.disabled)
-                await this._runSuite(suite);
-        }
+        try {
+            for (const suite of suites) {
+                if (!suite.disabled)
+                    await this._runSuite(suite);
+            }
 
+        } finally {
+            await this._finishRunAllSuites();
+        }
+    }
+
+    async _finishRunAllSuites() {
         const finalizeStartLabel = "runner-finalize-start";
         const finalizeEndLabel = "runner-finalize-end";
 
@@ -423,10 +441,11 @@ export class BenchmarkRunner {
     }
 
     async _runSuite(suite) {
-        const suitePrepareStartLabel = `suite-${suite.name}-prepare-start`;
-        const suitePrepareEndLabel = `suite-${suite.name}-prepare-end`;
-        const suiteStartLabel = `suite-${suite.name}-start`;
-        const suiteEndLabel = `suite-${suite.name}-end`;
+        const suiteName = suite.name;
+        const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
+        const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
+        const suiteStartLabel = `suite-${suiteName}-start`;
+        const suiteEndLabel = `suite-${suiteName}-end`;
 
         performance.mark(suitePrepareStartLabel);
         await this._prepareSuite(suite);
@@ -437,8 +456,18 @@ export class BenchmarkRunner {
             await this._runTestAndRecordResults(suite, test);
         performance.mark(suiteEndLabel);
 
-        performance.measure(`suite-${suite.name}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
-        performance.measure(`suite-${suite.name}`, suiteStartLabel, suiteEndLabel);
+        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
+        this._validateSuiteTotal(suiteName);
+    }
+
+    _validateSuiteTotal(suiteName) {
+        // When the test is fast and the precision is low (for example with Firefox'
+        // privacy.resistFingerprinting preference), it's possible that the measured
+        // total duration for an entire is 0.
+        const suiteTotal = this._measuredValues.tests[suiteName].total;
+        if (suiteTotal === 0)
+            throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`);
     }
 
     async _prepareSuite(suite) {

resources/main.css

@@ -423,6 +423,25 @@ section#summary > #confidence-number {
     color: var(--inactive-color);
 }
 
+section#summary.invalid h1,
+section#summary.invalid .gauge,
+section#summary.invalid hr {
+    opacity: 0.2;
+}
+section#summary.invalid #result-number {
+    color: var(--highlight);
+}
+section#summary.invalid .buttons {
+    display: none;
+}
+#invalid-score-text {
+    display: none;
+    text-align: center;
+}
+section#summary.invalid #invalid-score-text {
+    display: block;
+}
+
 section#details {
     --viewport-height: max(600px, 90vh);
     --viewport-width: max(800px, 80vw);

resources/main.mjs

@@ -107,19 +107,42 @@ class MainBenchmarkClient {
         this._metrics = metrics;
 
         const scoreResults = this._computeResults(this._measuredValuesList, "score");
-        this._updateGaugeNeedle(scoreResults.mean);
-        document.getElementById("result-number").textContent = scoreResults.formattedMean;
-        if (scoreResults.formattedDelta)
-            document.getElementById("confidence-number").textContent = `\u00b1 ${scoreResults.formattedDelta}`;
+        if (scoreResults.isValid)
+            this._populateValidScore(scoreResults);
+        else
+            this._populateInvalidScore();
 
         this._populateDetailedResults(metrics);
-
         if (params.developerMode)
             this.showResultsDetails();
         else
             this.showResultsSummary();
     }
 
+    handleError(error) {
+        console.assert(this._isRunning);
+        this._isRunning = false;
+        this._hasResults = true;
+        this._metrics = Object.create(null);
+        this._populateInvalidScore();
+        this.showResultsSummary();
+    }
+
+    _populateValidScore(scoreResults) {
+        document.getElementById("summary").className = "valid";
+
+        this._updateGaugeNeedle(scoreResults.mean);
+        document.getElementById("result-number").textContent = scoreResults.formattedMean;
+        if (scoreResults.formattedDelta)
+            document.getElementById("confidence-number").textContent = `\u00b1 ${scoreResults.formattedDelta}`;
+    }
+
+    _populateInvalidScore() {
+        document.getElementById("summary").className = "invalid";
+        document.getElementById("result-number").textContent = "Error";
+        document.getElementById("confidence-number").textContent = "";
+    }
+
     _computeResults(measuredValuesList, displayUnit) {
         function valueForUnit(measuredValues) {
             if (displayUnit === "ms")
@@ -137,9 +160,7 @@ class MainBenchmarkClient {
         }
 
         const values = measuredValuesList.map(valueForUnit);
-        const sum = values.reduce((a, b) => {
-            return a + b;
-        }, 0);
+        const sum = values.reduce((a, b) => a + b, 0);
         const arithmeticMean = sum / values.length;
         let meanSigFig = 4;
         let formattedDelta;
@@ -162,6 +183,7 @@ class MainBenchmarkClient {
             formattedMean: formattedMean,
             formattedDelta: formattedDelta,
             formattedMeanAndDelta: formattedMean + (formattedDelta ? ` \xb1 ${formattedDelta} (${formattedPercentDelta})` : ""),
+            isValid: values.length > 0 && isFinite(sum) && sum > 0,
         };
     }