Skip to content

Commit 066fb39

Browse files
panvapanva
committed
test: add WPTRunner support for variants and generating WPT reports
1 parent 9fafb0a commit 066fb39

3 files changed

Lines changed: 181 additions & 59 deletions

File tree

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,12 @@ test-message: test-build
595595
test-wpt: all
596596
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt
597597

598+
.PHONY: test-wpt-report
599+
test-wpt-report:
600+
$(RM) -r out/wpt
601+
mkdir -p out/wpt
602+
WPTREPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
603+
598604
.PHONY: test-simple
599605
test-simple: | cctest # Depends on 'all'.
600606
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential

‎test/common/wpt.js‎

Lines changed: 174 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,100 @@ const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
88
const events = require('events');
9+
const os = require('os');
910
const { inspect } = require('util');
1011
const { Worker } = require('worker_threads');
1112

13+
function getBrowserProperties() {
14+
const release = !!process.release.sourceUrl;
15+
const browser = {
16+
browser_channel: release ? 'stable' : 'experimental',
17+
browser_version: process.version,
18+
};
19+
20+
if (release) {
21+
browser.browser_version = process.version;
22+
} else if (process.env.NODE_REVISION != null) {
23+
browser.browser_version = process.version.replace('pre', process.env.NODE_REVISION);
24+
}
25+
26+
return browser;
27+
}
28+
29+
/**
30+
* Return one of three expected values
31+
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
32+
*/
33+
function getOs() {
34+
switch (os.type()) {
35+
case 'Linux':
36+
return 'linux';
37+
case 'Darwin':
38+
return 'mac';
39+
case 'Windows_NT':
40+
return 'win';
41+
default:
42+
throw new Error('Unsupported os.type()');
43+
}
44+
}
45+
46+
class WPTReport {
47+
constructor() {
48+
this.results = [];
49+
this.time_start = Date.now();
50+
}
51+
52+
addResult(name, status) {
53+
const result = {
54+
test: name,
55+
status,
56+
subtests: [],
57+
addSubtest(name, status, message) {
58+
const subtest = {
59+
status,
60+
name,
61+
};
62+
if (message) subtest.message = message;
63+
this.subtests.push(subtest);
64+
return subtest;
65+
},
66+
};
67+
this.results.push(result);
68+
return result;
69+
}
70+
71+
write() {
72+
this.time_end = Date.now();
73+
this.results = this.results.filter((result) => {
74+
return result.status === 'SKIP' || result.subtests.length !== 0;
75+
}).map((result) => {
76+
result.test = result.test.replace(/\.js(?:\?|$)/, '.html');
77+
return result;
78+
});
79+
80+
if (fs.existsSync('out/wpt/wptreport.json')) {
81+
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
82+
this.results = [...prev.results, ...this.results];
83+
this.time_start = prev.time_start;
84+
this.time_end = Math.max(this.time_end, prev.time_end);
85+
this.run_info = prev.run_info;
86+
} else {
87+
/**
88+
* Return required and some optional properties
89+
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
90+
*/
91+
this.run_info = {
92+
product: 'Node.js',
93+
...getBrowserProperties(),
94+
revision: process.env.WPT_REVISION || 'unknown',
95+
os: getOs(),
96+
};
97+
}
98+
99+
fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
100+
}
101+
}
102+
12103
// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
13104
// TODO: get rid of this half-baked harness in favor of the one
14105
// pulled from WPT
@@ -313,6 +404,10 @@ class WPTRunner {
313404
this.unexpectedFailures = [];
314405

315406
this.scriptsModifier = null;
407+
408+
if (process.env.WPTREPORT != null) {
409+
this.report = new WPTReport();
410+
}
316411
}
317412

318413
/**
@@ -339,18 +434,23 @@ class WPTRunner {
339434
this.scriptsModifier = modifier;
340435
}
341436

342-
get fullInitScript() {
437+
fullInitScript(locationSearchString) {
438+
let { initScript } = this;
439+
if (locationSearchString) {
440+
initScript = `${initScript}\n\n//===\nglobalThis.location &&= { search: "${locationSearchString}" };`;
441+
}
442+
343443
if (this.globalThisInitScripts.length === null) {
344-
return this.initScript;
444+
return initScript;
345445
}
346446

347447
const globalThisInitScript = this.globalThisInitScripts.join('\n\n//===\n');
348448

349-
if (this.initScript === null) {
449+
if (initScript === null) {
350450
return globalThisInitScript;
351451
}
352452

353-
return `${globalThisInitScript}\n\n//===\n${this.initScript}`;
453+
return `${globalThisInitScript}\n\n//===\n${initScript}`;
354454
}
355455

356456
/**
@@ -455,7 +555,7 @@ class WPTRunner {
455555
for (const spec of queue) {
456556
const testFileName = spec.filename;
457557
const content = spec.getContent();
458-
const meta = spec.title = this.getMeta(content);
558+
const meta = spec.meta = this.getMeta(content);
459559

460560
const absolutePath = spec.getAbsolutePath();
461561
const relativePath = spec.getRelativePath();
@@ -480,54 +580,65 @@ class WPTRunner {
480580
this.scriptsModifier?.(obj);
481581
scriptsToRun.push(obj);
482582

483-
const workerPath = path.join(__dirname, 'wpt/worker.js');
484-
const worker = new Worker(workerPath, {
485-
execArgv: this.flags,
486-
workerData: {
487-
testRelativePath: relativePath,
488-
wptRunner: __filename,
489-
wptPath: this.path,
490-
initScript: this.fullInitScript,
491-
harness: {
492-
code: fs.readFileSync(harnessPath, 'utf8'),
493-
filename: harnessPath,
583+
/**
584+
* Example test with no META variant
585+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
586+
*
587+
* Example test with multiple META variants
588+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
589+
*/
590+
for (const variant of meta.variant || ['']) {
591+
const workerPath = path.join(__dirname, 'wpt/worker.js');
592+
const worker = new Worker(workerPath, {
593+
execArgv: this.flags,
594+
workerData: {
595+
testRelativePath: relativePath,
596+
wptRunner: __filename,
597+
wptPath: this.path,
598+
initScript: this.fullInitScript(variant),
599+
harness: {
600+
code: fs.readFileSync(harnessPath, 'utf8'),
601+
filename: harnessPath,
602+
},
603+
scriptsToRun,
494604
},
495-
scriptsToRun,
496-
},
497-
});
498-
this.workers.set(testFileName, worker);
499-
500-
worker.on('message', (message) => {
501-
switch (message.type) {
502-
case 'result':
503-
return this.resultCallback(testFileName, message.result);
504-
case 'completion':
505-
return this.completionCallback(testFileName, message.status);
506-
default:
507-
throw new Error(`Unexpected message from worker: ${message.type}`);
508-
}
509-
});
605+
});
606+
this.workers.set(testFileName, worker);
607+
608+
let reportResult;
609+
worker.on('message', (message) => {
610+
switch (message.type) {
611+
case 'result':
612+
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
613+
return this.resultCallback(testFileName, message.result, reportResult);
614+
case 'completion':
615+
return this.completionCallback(testFileName, message.status);
616+
default:
617+
throw new Error(`Unexpected message from worker: ${message.type}`);
618+
}
619+
});
510620

511-
worker.on('error', (err) => {
512-
if (!this.inProgress.has(testFileName)) {
513-
// The test is already finished. Ignore errors that occur after it.
514-
// This can happen normally, for example in timers tests.
515-
return;
516-
}
517-
this.fail(
518-
testFileName,
519-
{
520-
status: NODE_UNCAUGHT,
521-
name: 'evaluation in WPTRunner.runJsTests()',
522-
message: err.message,
523-
stack: inspect(err),
524-
},
525-
kUncaught,
526-
);
527-
this.inProgress.delete(testFileName);
528-
});
621+
worker.on('error', (err) => {
622+
if (!this.inProgress.has(testFileName)) {
623+
// The test is already finished. Ignore errors that occur after it.
624+
// This can happen normally, for example in timers tests.
625+
return;
626+
}
627+
this.fail(
628+
testFileName,
629+
{
630+
status: NODE_UNCAUGHT,
631+
name: 'evaluation in WPTRunner.runJsTests()',
632+
message: err.message,
633+
stack: inspect(err),
634+
},
635+
kUncaught,
636+
);
637+
this.inProgress.delete(testFileName);
638+
});
529639

530-
await events.once(worker, 'exit').catch(() => {});
640+
await events.once(worker, 'exit').catch(() => {});
641+
}
531642
}
532643

533644
process.on('exit', () => {
@@ -587,6 +698,8 @@ class WPTRunner {
587698
}
588699
}
589700

701+
this.report?.write();
702+
590703
const ran = queue.length;
591704
const total = ran + skipped;
592705
const passed = ran - expectedFailures - failures.length;
@@ -611,8 +724,7 @@ class WPTRunner {
611724

612725
getTestTitle(filename) {
613726
const spec = this.specMap.get(filename);
614-
const title = spec.meta && spec.meta.title;
615-
return title ? `${filename} : ${title}` : filename;
727+
return spec.meta?.title || filename;
616728
}
617729

618730
// Map WPT test status to strings
@@ -638,14 +750,14 @@ class WPTRunner {
638750
* @param {string} filename
639751
* @param {Test} test The Test object returned by WPT harness
640752
*/
641-
resultCallback(filename, test) {
753+
resultCallback(filename, test, reportResult) {
642754
const status = this.getTestStatus(test.status);
643755
const title = this.getTestTitle(filename);
644756
console.log(`---- ${title} ----`);
645757
if (status !== kPass) {
646-
this.fail(filename, test, status);
758+
this.fail(filename, test, status, reportResult);
647759
} else {
648-
this.succeed(filename, test, status);
760+
this.succeed(filename, test, status, reportResult);
649761
}
650762
}
651763

@@ -693,11 +805,12 @@ class WPTRunner {
693805
}
694806
}
695807

696-
succeed(filename, test, status) {
808+
succeed(filename, test, status, reportResult) {
697809
console.log(`[${status.toUpperCase()}] ${test.name}`);
810+
reportResult?.addSubtest(test.name, 'PASS');
698811
}
699812

700-
fail(filename, test, status) {
813+
fail(filename, test, status, reportResult) {
701814
const spec = this.specMap.get(filename);
702815
const expected = spec.failedTests.includes(test.name);
703816
if (expected) {
@@ -713,6 +826,9 @@ class WPTRunner {
713826
const command = `${process.execPath} ${process.execArgv}` +
714827
` ${require.main.filename} ${filename}`;
715828
console.log(`Command: ${command}\n`);
829+
830+
reportResult?.addSubtest(test.name, 'FAIL', test.message);
831+
716832
this.addTestResult(filename, {
717833
name: test.name,
718834
expected,
@@ -742,7 +858,7 @@ class WPTRunner {
742858
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
743859
const key = parts[1];
744860
const value = parts[2];
745-
if (key === 'script') {
861+
if (key === 'script' || key === 'variant') {
746862
if (result[key]) {
747863
result[key].push(value);
748864
} else {

‎test/wpt/test-webcrypto.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ const { WPTRunner } = require('../common/wpt');
99
const runner = new WPTRunner('WebCryptoAPI');
1010

1111
runner.setInitScript(`
12-
global.location = {};
12+
globalThis.location ||= {};
1313
`);
1414

1515
runner.pretendGlobalThisAs('Window');

0 commit comments

Comments
 (0)
⚡