Remove code using the listing_meta.json file. (#4680)
* Remove code using the listing_meta.json file. This CL removes the infrastructure which validates and uses the listing_meta.json file. The listing_meta.json file itself will be removed in a followup CL. Issue: #4592 * remove chunked2sec * run lint fix * fix lint warnings * Restore case count report
This commit is contained in:
dan sinclair
GitHub
parent
1aae59469a
commit
1d676e4895
10 changed files with 19 additions and 517 deletions
10
Gruntfile.js
10
Gruntfile.js
|
|
@ -58,10 +58,6 @@ module.exports = function (grunt) {
|
|||
cmd: 'node',
|
||||
args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_unchunked.json'],
|
||||
},
|
||||
'write-out-wpt-cts-html-chunked2sec': {
|
||||
cmd: 'node',
|
||||
args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_chunked2sec.json'],
|
||||
},
|
||||
'write-out-wpt-cts-html-withsomeworkers': {
|
||||
cmd: 'node',
|
||||
args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_withsomeworkers.json'],
|
||||
|
|
@ -203,11 +199,7 @@ module.exports = function (grunt) {
|
|||
|
||||
concurrent: {
|
||||
'write-out-wpt-cts-html-all': {
|
||||
tasks: [
|
||||
'run:write-out-wpt-cts-html',
|
||||
'run:write-out-wpt-cts-html-chunked2sec',
|
||||
'run:write-out-wpt-cts-html-withsomeworkers',
|
||||
],
|
||||
tasks: ['run:write-out-wpt-cts-html', 'run:write-out-wpt-cts-html-withsomeworkers'],
|
||||
},
|
||||
'all-builds': {
|
||||
tasks: ['build-standalone', 'build-wpt', 'run:build-out-node'],
|
||||
|
|
|
|||
|
|
@ -1,148 +0,0 @@
|
|||
# Adding Timing Metadata
|
||||
|
||||
## listing_meta.json files
|
||||
|
||||
`listing_meta.json` files are SEMI AUTO-GENERATED.
|
||||
|
||||
The raw data may be edited manually, to add entries or change timing values.
|
||||
|
||||
The list of tests in this file is **not** guaranteed to stay up to date.
|
||||
Use the generated `gen/*_variant_list*.json` if you need a complete list.
|
||||
|
||||
The `subcaseMS` values are estimates. They can be set to 0 or omitted if for some reason
|
||||
you can't estimate the time (or there's an existing test with a long name and
|
||||
slow subcases that would result in query strings that are too long).
|
||||
It's OK if the number is estimated too high.
|
||||
|
||||
These entries are estimates for the amount of time that subcases take to run,
|
||||
and are used as inputs into the WPT tooling to attempt to portion out tests into
|
||||
approximately same-sized chunks. High estimates are OK, they just may generate
|
||||
more chunks than necessary.
|
||||
|
||||
To check for missing or 0 entries, run
|
||||
`tools/validate --print-metadata-warnings src/webgpu`
|
||||
and look at the resulting warnings.
|
||||
|
||||
### Performance
|
||||
|
||||
Note this data is typically captured by developers using higher-end
|
||||
computers, so typical test machines might execute more slowly. For this
|
||||
reason, the WPT chunking should be configured to generate chunks much shorter
|
||||
than 5 seconds (a typical default time limit in WPT test executors) so they
|
||||
should still execute in under 5 seconds on lower-end computers.
|
||||
|
||||
## Problem
|
||||
|
||||
When renaming or removing tests from the CTS you will see an error like this
|
||||
when running `npm test` or `npm run standalone`:
|
||||
|
||||
```
|
||||
ERROR: Non-existent tests found in listing_meta.json. Please update:
|
||||
webgpu:api,operation,adapter,requestAdapter:old_test_that_got_renamed:*
|
||||
```
|
||||
|
||||
## Solution
|
||||
|
||||
This means there is a stale line in `src/webgpu/listing_meta.json` that needs
|
||||
to be deleted, or updated to match the rename that you did.
|
||||
|
||||
## Problem
|
||||
|
||||
You run `tools/validate --print-metadata-warnings src/webgpu`
|
||||
and want to fix the warnings.
|
||||
|
||||
## Solution 1 (manual, best for one-off updates of simple tests)
|
||||
|
||||
If you're developing new tests and need to update this file, it is sometimes
|
||||
easiest to do so manually. Run your tests under your usual development workflow
|
||||
and see how long they take. In the standalone web runner `npm start`, the total
|
||||
time for a test case is reported on the right-hand side when the case logs are
|
||||
expanded.
|
||||
|
||||
Record the average time per *subcase* across all cases of the test (you may need
|
||||
to compute this) into the `listing_meta.json` file.
|
||||
|
||||
## Solution 2 (semi-automated)
|
||||
|
||||
There exists tooling in the CTS repo for generating appropriate estimates for
|
||||
these values, though they do require some manual intervention. The rest of this
|
||||
doc will be a walkthrough of running these tools.
|
||||
|
||||
Timing data can be captured in bulk and "merged" into this file using
|
||||
the `merge_listing_times` tool. This is
|
||||
This is useful when a large number of tests
|
||||
change or otherwise a lot of tests need to be updated, but it also automates the
|
||||
manual steps above.
|
||||
|
||||
The tool can also be used without any inputs to reformat `listing_meta.json`.
|
||||
Please read the help message of `merge_listing_times` for more information.
|
||||
|
||||
### Websocket Logger
|
||||
|
||||
The first tool that needs to be run is `websocket-logger`, which receives data
|
||||
on a WebSocket channel at `localhost:59497` to capture timing data when CTS is run. This
|
||||
should be run in a separate process/terminal, since it needs to stay running
|
||||
throughout the following steps.
|
||||
|
||||
In the `tools/websocket-logger/` directory:
|
||||
|
||||
```
|
||||
npm ci
|
||||
npm start
|
||||
```
|
||||
|
||||
The output from this command will indicate where the results are being logged,
|
||||
which will be needed later. For example:
|
||||
|
||||
```
|
||||
...
|
||||
Writing to wslog-2023-09-12T18-57-34.txt
|
||||
...
|
||||
```
|
||||
|
||||
See also [tools/websocket-logger/README.md](../tools/websocket-logger/README.md).
|
||||
|
||||
### Running CTS
|
||||
|
||||
Now we need to run the specific cases in CTS that we need to time.
|
||||
|
||||
This should be possible under any development workflow by logging through a
|
||||
side-channel (as long as its runtime environment, like Node, supports WebSockets).
|
||||
Regardless of development workflow, you need to enable logToWebSocket flag
|
||||
(`?log_to_web_socket=1` in browser, `--log-to-web-socket` on command line, or
|
||||
just hack it in by switching the default in `options.ts`).
|
||||
|
||||
The most well-tested way to do this is using the standalone web runner.
|
||||
|
||||
This requires serving the CTS locally. In the project root:
|
||||
|
||||
```
|
||||
npm run standalone
|
||||
npm start
|
||||
```
|
||||
|
||||
Once this is started you can then direct a WebGPU enabled browser to the
|
||||
specific CTS entry and run the tests, for example:
|
||||
|
||||
```
|
||||
http://localhost:8080/standalone/?log_to_web_socket=1&q=webgpu:*
|
||||
```
|
||||
|
||||
If the tests have a high variance in runtime, you can run them multiple times.
|
||||
The longest recorded time will be used.
|
||||
|
||||
### Merging metadata
|
||||
|
||||
The final step is to merge the new data that has been captured into the JSON
|
||||
file.
|
||||
|
||||
This can be done using the following command:
|
||||
|
||||
```
|
||||
tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-2023-09-12T18-57-34.txt
|
||||
tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-*.txt
|
||||
```
|
||||
|
||||
Or, you can point it to one of the log files from a specific invocation of websocket-logger.
|
||||
|
||||
Now you just need to commit the pending diff in your repo.
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
import { assert } from '../util/util.js';
|
||||
|
||||
/** Metadata about tests (that can't be derived at runtime). */
|
||||
export type TestMetadata = {
|
||||
/**
|
||||
* Estimated average time-per-subcase, in milliseconds.
|
||||
* This is used to determine chunking granularity when exporting to WPT with
|
||||
* chunking enabled (like out-wpt/cts-chunked2sec.https.html).
|
||||
*/
|
||||
subcaseMS: number;
|
||||
};
|
||||
|
||||
export type TestMetadataListing = {
|
||||
[testQuery: string]: TestMetadata;
|
||||
};
|
||||
|
||||
export function loadMetadataForSuite(suiteDir: string): TestMetadataListing | null {
|
||||
assert(typeof require !== 'undefined', 'loadMetadataForSuite is only implemented on Node');
|
||||
/* eslint-disable-next-line n/no-restricted-require */
|
||||
const fs = require('fs');
|
||||
|
||||
const metadataFile = `${suiteDir}/listing_meta.json`;
|
||||
if (!fs.existsSync(metadataFile)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const metadata: TestMetadataListing = JSON.parse(fs.readFileSync(metadataFile, 'utf8'));
|
||||
return metadata;
|
||||
}
|
||||
|
|
@ -74,8 +74,7 @@ export abstract class TestFileLoader extends EventTarget {
|
|||
{
|
||||
subqueriesToExpand = [],
|
||||
fullyExpandSubtrees = [],
|
||||
maxChunkTime = Infinity,
|
||||
}: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[]; maxChunkTime?: number } = {}
|
||||
}: { subqueriesToExpand?: string[]; fullyExpandSubtrees?: string[] } = {}
|
||||
): Promise<TestTree> {
|
||||
const tree = await loadTreeForQuery(this, query, {
|
||||
subqueriesToExpand: subqueriesToExpand.map(s => {
|
||||
|
|
@ -84,7 +83,6 @@ export abstract class TestFileLoader extends EventTarget {
|
|||
return q;
|
||||
}),
|
||||
fullyExpandSubtrees: fullyExpandSubtrees.map(s => parseQuery(s)),
|
||||
maxChunkTime,
|
||||
});
|
||||
this.dispatchEvent(new MessageEvent<void>('finish'));
|
||||
return tree;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { loadMetadataForSuite, TestMetadataListing } from '../framework/metadata.js';
|
||||
import { globalTestConfig } from '../framework/test_config.js';
|
||||
import { RunCase, RunFn } from '../internal/test_group.js';
|
||||
import { assert, now } from '../util/util.js';
|
||||
|
|
@ -101,21 +100,8 @@ export class TestTree {
|
|||
);
|
||||
}
|
||||
|
||||
static async create(
|
||||
forQuery: TestQuery,
|
||||
root: TestSubtree,
|
||||
maxChunkTime: number
|
||||
): Promise<TestTree> {
|
||||
const suite = forQuery.suite;
|
||||
|
||||
let chunking = undefined;
|
||||
if (Number.isFinite(maxChunkTime)) {
|
||||
const metadata = loadMetadataForSuite(`./src/${suite}`);
|
||||
assert(metadata !== null, `metadata for ${suite} is missing, but maxChunkTime was requested`);
|
||||
chunking = { metadata, maxChunkTime };
|
||||
}
|
||||
await TestTree.propagateCounts(root, chunking);
|
||||
|
||||
static async create(forQuery: TestQuery, root: TestSubtree): Promise<TestTree> {
|
||||
await TestTree.propagateCounts(root);
|
||||
return new TestTree(forQuery, root);
|
||||
}
|
||||
|
||||
|
|
@ -206,14 +192,13 @@ export class TestTree {
|
|||
|
||||
/** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */
|
||||
static async propagateCounts(
|
||||
subtree: TestSubtree,
|
||||
chunking: { metadata: TestMetadataListing; maxChunkTime: number } | undefined
|
||||
subtree: TestSubtree
|
||||
): Promise<{ tests: number; nodesWithTODO: number; totalTimeMS: number; subcaseCount: number }> {
|
||||
subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0, totalTimeMS: 0 };
|
||||
subtree.subcaseCount = 0;
|
||||
for (const [, child] of subtree.children) {
|
||||
if ('children' in child) {
|
||||
const counts = await TestTree.propagateCounts(child, chunking);
|
||||
const counts = await TestTree.propagateCounts(child);
|
||||
subtree.subtreeCounts.tests += counts.tests;
|
||||
subtree.subtreeCounts.nodesWithTODO += counts.nodesWithTODO;
|
||||
subtree.subtreeCounts.totalTimeMS += counts.totalTimeMS;
|
||||
|
|
@ -223,32 +208,6 @@ export class TestTree {
|
|||
}
|
||||
}
|
||||
|
||||
// If we're chunking based on a maxChunkTime, then at each
|
||||
// TestQueryMultiCase node of the tree we look at its total time. If the
|
||||
// total time is larger than the maxChunkTime, we set collapsible=false to
|
||||
// make sure it gets split up in the output. Note:
|
||||
// - TestQueryMultiTest and higher nodes are never set to collapsible anyway, so we ignore them.
|
||||
// - TestQuerySingleCase nodes can't be collapsed, so we ignore them.
|
||||
if (chunking && subtree.query instanceof TestQueryMultiCase) {
|
||||
const testLevelQuery = new TestQueryMultiCase(
|
||||
subtree.query.suite,
|
||||
subtree.query.filePathParts,
|
||||
subtree.query.testPathParts,
|
||||
{}
|
||||
).toString();
|
||||
|
||||
const metadata = chunking.metadata;
|
||||
|
||||
const subcaseTiming: number | undefined = metadata[testLevelQuery]?.subcaseMS;
|
||||
if (subcaseTiming !== undefined) {
|
||||
const totalTiming = subcaseTiming * subtree.subcaseCount;
|
||||
subtree.subtreeCounts.totalTimeMS = totalTiming;
|
||||
if (totalTiming > chunking.maxChunkTime) {
|
||||
subtree.collapsible = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { ...subtree.subtreeCounts, subcaseCount: subtree.subcaseCount ?? 0 };
|
||||
}
|
||||
|
||||
|
|
@ -287,8 +246,7 @@ export async function loadTreeForQuery(
|
|||
{
|
||||
subqueriesToExpand,
|
||||
fullyExpandSubtrees = [],
|
||||
maxChunkTime = Infinity,
|
||||
}: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[]; maxChunkTime?: number }
|
||||
}: { subqueriesToExpand: TestQuery[]; fullyExpandSubtrees?: TestQuery[] }
|
||||
): Promise<TestTree> {
|
||||
const suite = queryToLoad.suite;
|
||||
const specs = await loader.listing(suite);
|
||||
|
|
@ -454,7 +412,7 @@ export async function loadTreeForQuery(
|
|||
}
|
||||
assert(foundCase, `Query \`${queryToLoad.toString()}\` does not match any cases`);
|
||||
|
||||
return TestTree.create(queryToLoad, subtreeL0, maxChunkTime);
|
||||
return TestTree.create(queryToLoad, subtreeL0);
|
||||
}
|
||||
|
||||
function setSubtreeDescriptionAndCountTODOs(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
import { loadMetadataForSuite } from '../framework/metadata.js';
|
||||
import { SpecFile } from '../internal/file_loader.js';
|
||||
import { TestQueryMultiCase, TestQueryMultiFile } from '../internal/query/query.js';
|
||||
import { validQueryPart } from '../internal/query/validQueryPart.js';
|
||||
|
|
@ -47,11 +46,7 @@ async function crawlFilesRecursively(dir: string): Promise<string[]> {
|
|||
|
||||
export async function crawl(
|
||||
suiteDir: string,
|
||||
opts: {
|
||||
validate: boolean;
|
||||
printMetadataWarnings: boolean;
|
||||
printCaseCountReport: boolean;
|
||||
} | null = null
|
||||
opts: { validate: boolean; printCaseCountReport: boolean } | null = null
|
||||
): Promise<TestSuiteListingEntry[]> {
|
||||
if (!fs.existsSync(suiteDir)) {
|
||||
throw new Error(`Could not find suite: ${suiteDir}`);
|
||||
|
|
@ -60,17 +55,6 @@ export async function crawl(
|
|||
let totalCases = 0;
|
||||
let totalSubcases = 0;
|
||||
|
||||
let validateTimingsEntries;
|
||||
if (opts?.validate) {
|
||||
const metadata = loadMetadataForSuite(suiteDir);
|
||||
if (metadata) {
|
||||
validateTimingsEntries = {
|
||||
metadata,
|
||||
testsFoundInFiles: new Set<string>(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Crawl files and convert paths to be POSIX-style, relative to suiteDir.
|
||||
const filesToEnumerate = (await crawlFilesRecursively(suiteDir))
|
||||
.map(f => path.relative(suiteDir, f).replace(/\\/g, '/'))
|
||||
|
|
@ -83,9 +67,8 @@ export async function crawl(
|
|||
const filepathWithoutExtension = file.substring(0, file.length - specFileSuffix.length);
|
||||
const pathSegments = filepathWithoutExtension.split('/');
|
||||
|
||||
const suite = path.basename(suiteDir);
|
||||
|
||||
if (opts?.validate) {
|
||||
const suite = path.basename(suiteDir);
|
||||
const filename = `../../${suite}/${filepathWithoutExtension}.spec.js`;
|
||||
|
||||
assert(!process.env.STANDALONE_DEV_SERVER);
|
||||
|
|
@ -95,7 +78,7 @@ export async function crawl(
|
|||
|
||||
mod.g.validate(new TestQueryMultiFile(suite, pathSegments));
|
||||
|
||||
if (opts?.printCaseCountReport || validateTimingsEntries) {
|
||||
if (opts?.printCaseCountReport) {
|
||||
for (const t of mod.g.iterate()) {
|
||||
const testQuery = new TestQueryMultiCase(
|
||||
suite,
|
||||
|
|
@ -108,21 +91,13 @@ export async function crawl(
|
|||
let subcases = 0;
|
||||
for (const c of t.iterate(null)) {
|
||||
cases++;
|
||||
if (opts?.printCaseCountReport) {
|
||||
subcases += c.computeSubcaseCount();
|
||||
}
|
||||
subcases += c.computeSubcaseCount();
|
||||
}
|
||||
|
||||
if (opts?.printCaseCountReport) {
|
||||
const perCase = (subcases / cases).toFixed(0);
|
||||
console.log(`${testQuery} - ${cases} cases, ${subcases} subcases (~${perCase}/case)`);
|
||||
totalCases += cases;
|
||||
totalSubcases += subcases;
|
||||
}
|
||||
|
||||
if (validateTimingsEntries && cases > 0) {
|
||||
validateTimingsEntries.testsFoundInFiles.add(testQuery);
|
||||
}
|
||||
const perCase = (subcases / cases).toFixed(0);
|
||||
console.log(`${testQuery} - ${cases} cases, ${subcases} subcases (~${perCase}/case)`);
|
||||
totalCases += cases;
|
||||
totalSubcases += subcases;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -130,6 +105,7 @@ export async function crawl(
|
|||
for (const p of pathSegments) {
|
||||
assert(validQueryPart.test(p), `Invalid directory name ${p}; must match ${validQueryPart}`);
|
||||
}
|
||||
|
||||
entries.push({ file: pathSegments });
|
||||
} else if (path.basename(file) === 'README.txt') {
|
||||
const dirname = path.dirname(file);
|
||||
|
|
@ -142,59 +118,10 @@ export async function crawl(
|
|||
}
|
||||
}
|
||||
|
||||
if (validateTimingsEntries) {
|
||||
const zeroEntries = [];
|
||||
const staleEntries = [];
|
||||
for (const [metadataKey, metadataValue] of Object.entries(validateTimingsEntries.metadata)) {
|
||||
if (metadataKey.startsWith('_')) {
|
||||
// Ignore json "_comments".
|
||||
continue;
|
||||
}
|
||||
if (metadataValue.subcaseMS <= 0) {
|
||||
zeroEntries.push(metadataKey);
|
||||
}
|
||||
if (!validateTimingsEntries.testsFoundInFiles.has(metadataKey)) {
|
||||
staleEntries.push(metadataKey);
|
||||
}
|
||||
}
|
||||
if (zeroEntries.length && opts?.printMetadataWarnings) {
|
||||
console.warn(
|
||||
'WARNING: subcaseMS ≤ 0 found in listing_meta.json (see docs/adding_timing_metadata.md):'
|
||||
);
|
||||
for (const metadataKey of zeroEntries) {
|
||||
console.warn(` ${metadataKey}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (opts?.printMetadataWarnings) {
|
||||
const missingEntries = [];
|
||||
for (const metadataKey of validateTimingsEntries.testsFoundInFiles) {
|
||||
if (!(metadataKey in validateTimingsEntries.metadata)) {
|
||||
missingEntries.push(metadataKey);
|
||||
}
|
||||
}
|
||||
if (missingEntries.length) {
|
||||
console.error(
|
||||
'WARNING: Tests missing from listing_meta.json (see docs/adding_timing_metadata.md):'
|
||||
);
|
||||
for (const metadataKey of missingEntries) {
|
||||
console.error(` ${metadataKey}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (staleEntries.length) {
|
||||
console.error('ERROR: Non-existent tests found in listing_meta.json. Please update:');
|
||||
for (const metadataKey of staleEntries) {
|
||||
console.error(` ${metadataKey}`);
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
}
|
||||
|
||||
if (opts?.printCaseCountReport) {
|
||||
console.log(`-----\nTOTAL: ${totalCases} cases, ${totalSubcases} subcases`);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ gen_wpt_cts_html.ts. Example:
|
|||
"out": "path/to/output/cts.https.html",
|
||||
"outJSON": "path/to/output/webgpu_variant_list.json",
|
||||
"template": "path/to/template/cts.https.html",
|
||||
"maxChunkTimeMS": 2000
|
||||
}
|
||||
|
||||
Usage (advanced) (deprecated, use config file):
|
||||
|
|
@ -63,14 +62,6 @@ interface ConfigJSON {
|
|||
outVariantList?: string;
|
||||
/** Input template filename, relative to config file. */
|
||||
template: string;
|
||||
/**
|
||||
* Maximum time for a single WPT "variant" chunk, in milliseconds. Defaults to infinity.
|
||||
*
|
||||
* This data is typically captured by developers on higher-end computers, so typical test
|
||||
* machines might execute more slowly. For this reason, use a time much less than 5 seconds
|
||||
* (a typical default time limit in WPT test executors).
|
||||
*/
|
||||
maxChunkTimeMS?: number;
|
||||
/**
|
||||
* List of argument prefixes (what comes before the test query), and optionally a list of
|
||||
* test queries to run under that prefix. Defaults to `['?q=']`.
|
||||
|
|
@ -97,7 +88,6 @@ interface Config {
|
|||
out: string;
|
||||
outVariantList?: string;
|
||||
template: string;
|
||||
maxChunkTimeMS: number;
|
||||
argumentsPrefixes: ArgumentsPrefixConfig[];
|
||||
noLongPathAssert: boolean;
|
||||
expectations?: {
|
||||
|
|
@ -144,7 +134,6 @@ let config: Config;
|
|||
suite: configJSON.suite,
|
||||
out: path.resolve(jsonFileDir, configJSON.out),
|
||||
template: path.resolve(jsonFileDir, configJSON.template),
|
||||
maxChunkTimeMS: configJSON.maxChunkTimeMS ?? Infinity,
|
||||
argumentsPrefixes: configJSON.argumentsPrefixes
|
||||
? [...reifyArgumentsPrefixesConfig(configJSON.argumentsPrefixes)]
|
||||
: [{ prefix: '?q=' }],
|
||||
|
|
@ -185,7 +174,6 @@ let config: Config;
|
|||
suite,
|
||||
out: outFile,
|
||||
template: templateFile,
|
||||
maxChunkTimeMS: Infinity,
|
||||
argumentsPrefixes: [{ prefix: '?q=' }],
|
||||
noLongPathAssert: false,
|
||||
};
|
||||
|
|
@ -206,8 +194,6 @@ let config: Config;
|
|||
printUsageAndExit(1);
|
||||
}
|
||||
|
||||
const useChunking = Number.isFinite(config.maxChunkTimeMS);
|
||||
|
||||
// Sort prefixes from longest to shortest
|
||||
config.argumentsPrefixes.sort((a, b) => b.prefix.length - a.prefix.length);
|
||||
|
||||
|
|
@ -247,19 +233,17 @@ let config: Config;
|
|||
const tree = await loader.loadTree(rootQuery, {
|
||||
subqueriesToExpand,
|
||||
fullyExpandSubtrees: fullyExpand.get(prefix),
|
||||
maxChunkTime: config.maxChunkTimeMS,
|
||||
});
|
||||
|
||||
lines.push(undefined); // output blank line between prefixes
|
||||
const prefixComment = { comment: `Prefix: "${prefix}"` }; // contents will be updated later
|
||||
if (useChunking) lines.push(prefixComment);
|
||||
|
||||
const filesSeen = new Set<string>();
|
||||
const testsSeen = new Set<string>();
|
||||
let variantCount = 0;
|
||||
|
||||
const alwaysExpandThroughLevel = 2; // expand to, at minimum, every test.
|
||||
loopOverNodes: for (const { query, subtreeCounts } of tree.iterateCollapsedNodes({
|
||||
loopOverNodes: for (const { query } of tree.iterateCollapsedNodes({
|
||||
alwaysExpandThroughLevel,
|
||||
})) {
|
||||
assert(query instanceof TestQueryMultiCase);
|
||||
|
|
@ -288,7 +272,6 @@ let config: Config;
|
|||
|
||||
lines.push({
|
||||
urlQueryString: prefix + query.toString(), // "?debug=0&q=..."
|
||||
comment: useChunking ? `estimated: ${subtreeCounts?.totalTimeMS.toFixed(3)} ms` : undefined,
|
||||
});
|
||||
|
||||
variantCount++;
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
import * as fs from 'fs';
|
||||
import * as process from 'process';
|
||||
import * as readline from 'readline';
|
||||
|
||||
import { TestMetadataListing } from '../framework/metadata.js';
|
||||
import { parseQuery } from '../internal/query/parseQuery.js';
|
||||
import { TestQueryMultiCase, TestQuerySingleCase } from '../internal/query/query.js';
|
||||
import { CaseTimingLogLine } from '../internal/test_group.js';
|
||||
import { assert } from '../util/util.js';
|
||||
|
||||
// For information on listing_meta.json file maintenance, please read
|
||||
// tools/merge_listing_times first.
|
||||
|
||||
function usage(rc: number): never {
|
||||
console.error(`Usage: tools/merge_listing_times [options] SUITES... -- [TIMING_LOG_FILES...]
|
||||
|
||||
Options:
|
||||
--help Print this message and exit.
|
||||
|
||||
Reads raw case timing data for each suite in SUITES, from all TIMING_LOG_FILES
|
||||
(see below), and merges it into the src/*/listing_meta.json files checked into
|
||||
the repository. The timing data in the listing_meta.json files is updated with
|
||||
the newly-observed timing data *if the new timing is slower*. That is, it will
|
||||
only increase the values in the listing_meta.json file, and will only cause WPT
|
||||
chunks to become smaller.
|
||||
|
||||
If there are no TIMING_LOG_FILES, this just regenerates (reformats) the file
|
||||
using the data already present.
|
||||
|
||||
In more detail:
|
||||
|
||||
- Reads per-case timing data in any of the SUITES, from all TIMING_LOG_FILES
|
||||
(ignoring skipped cases), and averages it over the number of subcases.
|
||||
In the case of cases that have run multiple times, takes the max of each.
|
||||
- Compiles the average time-per-subcase for each test seen.
|
||||
- For each suite seen, loads its listing_meta.json, takes the max of the old and
|
||||
new data, and writes it back out.
|
||||
|
||||
See 'docs/adding_timing_metadata.md' for how to generate TIMING_LOG_FILES files.
|
||||
`);
|
||||
process.exit(rc);
|
||||
}
|
||||
|
||||
const kHeader = `{
|
||||
"_comment": "SEMI AUTO-GENERATED. This list is NOT exhaustive. Please read docs/adding_timing_metadata.md.",
|
||||
`;
|
||||
const kFooter = `\
|
||||
"_end": ""
|
||||
}
|
||||
`;
|
||||
|
||||
const argv = process.argv;
|
||||
if (argv.some(v => v.startsWith('-') && v !== '--') || argv.every(v => v !== '--')) {
|
||||
usage(0);
|
||||
}
|
||||
const suites = [];
|
||||
const timingLogFilenames = [];
|
||||
let seenDashDash = false;
|
||||
for (const arg of argv.slice(2)) {
|
||||
if (arg === '--') {
|
||||
seenDashDash = true;
|
||||
continue;
|
||||
} else if (arg.startsWith('-')) {
|
||||
usage(0);
|
||||
}
|
||||
|
||||
if (seenDashDash) {
|
||||
timingLogFilenames.push(arg);
|
||||
} else {
|
||||
suites.push(arg);
|
||||
}
|
||||
}
|
||||
if (!seenDashDash) {
|
||||
usage(0);
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
// Read the log files to find the log line for each *case* query. If a case
|
||||
// ran multiple times, take the one with the largest average subcase time.
|
||||
const caseTimes = new Map<string, CaseTimingLogLine>();
|
||||
for (const timingLogFilename of timingLogFilenames) {
|
||||
const rl = readline.createInterface({
|
||||
input: fs.createReadStream(timingLogFilename),
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
|
||||
for await (const line of rl) {
|
||||
const parsed: CaseTimingLogLine = JSON.parse(line);
|
||||
|
||||
const prev = caseTimes.get(parsed.q);
|
||||
if (prev !== undefined) {
|
||||
const timePerSubcase = parsed.timems / Math.max(1, parsed.nonskippedSubcaseCount);
|
||||
const prevTimePerSubcase = prev.timems / Math.max(1, prev.nonskippedSubcaseCount);
|
||||
|
||||
if (timePerSubcase > prevTimePerSubcase) {
|
||||
caseTimes.set(parsed.q, parsed);
|
||||
}
|
||||
} else {
|
||||
caseTimes.set(parsed.q, parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Accumulate total times per test. Map of suite -> query -> {totalTimeMS, caseCount}.
|
||||
const testTimes = new Map<string, Map<string, { totalTimeMS: number; subcaseCount: number }>>();
|
||||
for (const suite of suites) {
|
||||
testTimes.set(suite, new Map());
|
||||
}
|
||||
for (const [caseQString, caseTime] of caseTimes) {
|
||||
const caseQ = parseQuery(caseQString);
|
||||
assert(caseQ instanceof TestQuerySingleCase);
|
||||
const suite = caseQ.suite;
|
||||
const suiteTestTimes = testTimes.get(suite);
|
||||
if (suiteTestTimes === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const testQ = new TestQueryMultiCase(suite, caseQ.filePathParts, caseQ.testPathParts, {});
|
||||
const testQString = testQ.toString();
|
||||
|
||||
const prev = suiteTestTimes.get(testQString);
|
||||
if (prev !== undefined) {
|
||||
prev.totalTimeMS += caseTime.timems;
|
||||
prev.subcaseCount += caseTime.nonskippedSubcaseCount;
|
||||
} else {
|
||||
suiteTestTimes.set(testQString, {
|
||||
totalTimeMS: caseTime.timems,
|
||||
subcaseCount: caseTime.nonskippedSubcaseCount,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const currentMetadata: TestMetadataListing = JSON.parse(
|
||||
fs.readFileSync(`./src/${suite}/listing_meta.json`, 'utf8')
|
||||
);
|
||||
|
||||
const metadata = { ...currentMetadata };
|
||||
for (const [testQString, { totalTimeMS, subcaseCount }] of testTimes.get(suite)!) {
|
||||
const avgTime = totalTimeMS / Math.max(1, subcaseCount);
|
||||
if (testQString in metadata) {
|
||||
metadata[testQString].subcaseMS = Math.max(metadata[testQString].subcaseMS, avgTime);
|
||||
} else {
|
||||
metadata[testQString] = { subcaseMS: avgTime };
|
||||
}
|
||||
}
|
||||
|
||||
writeListings(suite, metadata);
|
||||
}
|
||||
})();
|
||||
|
||||
function writeListings(suite: string, metadata: TestMetadataListing) {
|
||||
const output = fs.createWriteStream(`./src/${suite}/listing_meta.json`);
|
||||
try {
|
||||
output.write(kHeader);
|
||||
const keys = Object.keys(metadata).sort();
|
||||
for (const k of keys) {
|
||||
if (k.startsWith('_')) {
|
||||
// Ignore json "_comments".
|
||||
continue;
|
||||
}
|
||||
assert(k.indexOf('"') === -1);
|
||||
output.write(` "${k}": { "subcaseMS": ${metadata[k].subcaseMS.toFixed(3)} },\n`);
|
||||
}
|
||||
output.write(kFooter);
|
||||
} finally {
|
||||
output.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -32,13 +32,11 @@ if (args.indexOf('--help') !== -1) {
|
|||
usage(0);
|
||||
}
|
||||
|
||||
let printMetadataWarnings = false;
|
||||
let printCaseCountReport = false;
|
||||
const suiteDirs = [];
|
||||
for (const arg of args) {
|
||||
switch (arg) {
|
||||
case '--print-metadata-warnings':
|
||||
printMetadataWarnings = true;
|
||||
break;
|
||||
case '--print-case-count-report':
|
||||
printCaseCountReport = true;
|
||||
|
|
@ -56,7 +54,6 @@ if (suiteDirs.length === 0) {
|
|||
for (const suiteDir of suiteDirs) {
|
||||
void crawl(suiteDir, {
|
||||
validate: true,
|
||||
printMetadataWarnings,
|
||||
printCaseCountReport,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
"suite": "webgpu",
|
||||
"out": "../out-wpt/cts-chunked2sec.https.html",
|
||||
"outVariantList": "../gen/webgpu_variant_list_chunked2sec.json",
|
||||
"template": "../src/common/templates/cts.https.html",
|
||||
"maxChunkTimeMS": 2000
|
||||
}
|
||||
Loading…
Reference in a new issue