test: initialize test/wpt to run URL and console .js tests

This patch:

- Creates a new test suite `wpt` that can be used to run a subset
  of Web Platform Tests
- Adds a `WPTRunner` in `test/common/wpt.js` that can run the WPT
  subset in `test/fixtures/wpt` with a vm and the WPT harness
  while taking the status file in `test/wpt/status` into account.
  Here we use a new format of status file (in JSON) to handle specific
  requirements (like ICU requirements) in the tests and to handle
  expected failures and TODOs.
- Adds documentation on how the runner and the update automation works
- Runs the WHATWG URL tests and the console tests with the new test
  runner.

With this patch we eliminates the need of copy-pasting with manual
modifications to update a large chunk of our WPT subset previously
maintained in `test/parallel`. Now the tests run in `test/wpt` can
be automatically updated with `git node wpt` without modifications
by the actual WPT harness instead of our home-grown mock.

There are still a few URL tests left that need to be migrated in the
upstream to be placed in .js instead of .html - we currently still use
the legacy harness mock in the test files.

PR-URL: https://github.com/nodejs/node/pull/24035
Refs: https://github.com/nodejs/node/issues/23192
Reviewed-By: Daijiro Wachi <daijiro.wachi@gmail.com>

Makefile

@@ -496,6 +496,9 @@ test-debug: test-build
 test-message: test-build
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) message
 
+test-wpt: all
+	$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt
+
 test-simple: | cctest bench-addons-build  # Depends on 'all'.
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
 

test/common/README.md

@@ -772,12 +772,19 @@ Deletes and recreates the testing temporary directory.
 
 ## WPT Module
 
-The wpt.js module is a port of parts of
-[W3C testharness.js](https://github.com/w3c/testharness.js) for testing the
-Node.js
-[WHATWG URL API](https://nodejs.org/api/url.html#url_the_whatwg_url_api)
-implementation with tests from
-[W3C Web Platform Tests](https://github.com/w3c/web-platform-tests).
+### harness
+
+A legacy port of [Web Platform Tests][] harness.
+
+See the source code for definitions. Please avoid using it in new
+code - the current usage of this port in tests is being migrated to
+the original WPT harness, see [the WPT tests README][].
+
+### Class: WPTRunner
+
+A driver class for running WPT with the WPT harness in a vm.
+
+See [the WPT tests README][] for details.
 
 
 [<Array>]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array
@@ -792,3 +799,5 @@ implementation with tests from
 [`hijackstdio.hijackStdErr()`]: #hijackstderrlistener
 [`hijackstdio.hijackStdOut()`]: #hijackstdoutlistener
 [internationalization]: https://github.com/nodejs/node/wiki/Intl
+[Web Platform Tests]: https://github.com/web-platform-tests/wpt
+[the WPT tests README]: ../wpt/README.md

test/common/wpt.js

@@ -2,9 +2,17 @@
 'use strict';
 
 const assert = require('assert');
+const common = require('../common');
+const fixtures = require('../common/fixtures');
+const fs = require('fs');
+const fsPromises = fs.promises;
+const path = require('path');
+const vm = require('vm');
 
 // https://github.com/w3c/testharness.js/blob/master/testharness.js
-module.exports = {
+// TODO: get rid of this half-baked harness in favor of the one
+// pulled from WPT
+const harnessMock = {
   test: (fn, desc) => {
     try {
       fn();
@@ -28,3 +36,420 @@ module.exports = {
     assert.fail(`Reached unreachable code: ${desc}`);
   }
 };
+
+class ResourceLoader {
+  constructor(path) {
+    this.path = path;
+  }
+
+  fetch(url, asPromise = true) {
+    // We need to patch this to load the WebIDL parser
+    url = url.replace(
+      '/resources/WebIDLParser.js',
+      '/resources/webidl2/lib/webidl2.js'
+    );
+    const file = url.startsWith('/') ?
+      fixtures.path('wpt', url) :
+      fixtures.path('wpt', this.path, url);
+    if (asPromise) {
+      return fsPromises.readFile(file)
+        .then((data) => {
+          return {
+            ok: true,
+            json() { return JSON.parse(data.toString()); },
+            text() { return data.toString(); }
+          };
+        });
+    } else {
+      return fs.readFileSync(file, 'utf8');
+    }
+  }
+}
+
+class WPTTest {
+  /**
+   * @param {string} mod
+   * @param {string} filename
+   * @param {string[]} requires
+   * @param {string | undefined} failReason
+   * @param {string | undefined} skipReason
+   */
+  constructor(mod, filename, requires, failReason, skipReason) {
+    this.module = mod; // name of the WPT module, e.g. 'url'
+    this.filename = filename;  // name of the test file
+    this.requires = requires;
+    this.failReason = failReason;
+    this.skipReason = skipReason;
+  }
+
+  getAbsolutePath() {
+    return fixtures.path('wpt', this.module, this.filename);
+  }
+
+  getContent() {
+    return fs.readFileSync(this.getAbsolutePath(), 'utf8');
+  }
+
+  shouldSkip() {
+    return this.failReason || this.skipReason;
+  }
+
+  requireIntl() {
+    return this.requires.includes('intl');
+  }
+}
+
+class StatusLoader {
+  constructor(path) {
+    this.path = path;
+    this.loaded = false;
+    this.status = null;
+    /** @type {WPTTest[]} */
+    this.tests = [];
+  }
+
+  loadTest(file) {
+    let requires = [];
+    let failReason;
+    let skipReason;
+    if (this.status[file]) {
+      requires = this.status[file].requires || [];
+      failReason = this.status[file].fail;
+      skipReason = this.status[file].skip;
+    }
+    return new WPTTest(this.path, file, requires,
+                       failReason, skipReason);
+  }
+
+  load() {
+    const dir = path.join(__dirname, '..', 'wpt');
+    const statusFile = path.join(dir, 'status', `${this.path}.json`);
+    const result = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
+    this.status = result;
+
+    const list = fs.readdirSync(fixtures.path('wpt', this.path));
+
+    for (const file of list) {
+      this.tests.push(this.loadTest(file));
+    }
+    this.loaded = true;
+  }
+
+  get jsTests() {
+    return this.tests.filter((test) => test.filename.endsWith('.js'));
+  }
+}
+
+const PASSED = 1;
+const FAILED = 2;
+const SKIPPED = 3;
+
+class WPTRunner {
+  constructor(path) {
+    this.path = path;
+    this.resource = new ResourceLoader(path);
+    this.sandbox = null;
+    this.context = null;
+
+    this.globals = new Map();
+
+    this.status = new StatusLoader(path);
+    this.status.load();
+    this.tests = new Map(
+      this.status.jsTests.map((item) => [item.filename, item])
+    );
+
+    this.results = new Map();
+    this.inProgress = new Set();
+  }
+
+  /**
+   * Specify that certain global descriptors from the object
+   * should be defined in the vm
+   * @param {object} obj
+   * @param {string[]} names
+   */
+  copyGlobalsFromObject(obj, names) {
+    for (const name of names) {
+      const desc = Object.getOwnPropertyDescriptor(global, name);
+      this.globals.set(name, desc);
+    }
+  }
+
+  /**
+   * Specify that certain global descriptors should be defined in the vm
+   * @param {string} name
+   * @param {object} descriptor
+   */
+  defineGlobal(name, descriptor) {
+    this.globals.set(name, descriptor);
+  }
+
+  // TODO(joyeecheung): work with the upstream to port more tests in .html
+  // to .js.
+  runJsTests() {
+    // TODO(joyeecheung): it's still under discussion whether we should leave
+    // err.name alone. See https://github.com/nodejs/node/issues/20253
+    const internalErrors = require('internal/errors');
+    internalErrors.useOriginalName = true;
+
+    let queue = [];
+
+    // If the tests are run as `node test/wpt/test-something.js subset.any.js`,
+    // only `subset.any.js` will be run by the runner.
+    if (process.argv[2]) {
+      const filename = process.argv[2];
+      if (!this.tests.has(filename)) {
+        throw new Error(`${filename} not found!`);
+      }
+      queue.push(this.tests.get(filename));
+    } else {
+      queue = this.buildQueue();
+    }
+
+    this.inProgress = new Set(queue.map((item) => item.filename));
+
+    for (const test of queue) {
+      const filename = test.filename;
+      const content = test.getContent();
+      const meta = test.title = this.getMeta(content);
+
+      const absolutePath = test.getAbsolutePath();
+      const context = this.generateContext(test.filename);
+      const code = this.mergeScripts(meta, content);
+      try {
+        vm.runInContext(code, context, {
+          filename: absolutePath
+        });
+      } catch (err) {
+        this.fail(filename, {
+          name: '',
+          message: err.message,
+          stack: err.stack
+        }, 'UNCAUGHT');
+        this.inProgress.delete(filename);
+      }
+    }
+    this.tryFinish();
+  }
+
+  mock() {
+    const resource = this.resource;
+    const result = {
+      // This is a mock, because at the moment fetch is not implemented
+      // in Node.js, but some tests and harness depend on this to pull
+      // resources.
+      fetch(file) {
+        return resource.fetch(file);
+      },
+      location: {},
+      GLOBAL: {
+        isWindow() { return false; }
+      },
+      Object
+    };
+
+    return result;
+  }
+
+  // Note: this is how our global space for the WPT test should look like
+  getSandbox() {
+    const result = this.mock();
+    for (const [name, desc] of this.globals) {
+      Object.defineProperty(result, name, desc);
+    }
+    return result;
+  }
+
+  generateContext(filename) {
+    const sandbox = this.sandbox = this.getSandbox();
+    const context = this.context = vm.createContext(sandbox);
+
+    const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
+    const harness = fs.readFileSync(harnessPath, 'utf8');
+    vm.runInContext(harness, context, {
+      filename: harnessPath
+    });
+
+    sandbox.add_result_callback(
+      this.resultCallback.bind(this, filename)
+    );
+    sandbox.add_completion_callback(
+      this.completionCallback.bind(this, filename)
+    );
+    sandbox.self = sandbox;
+    // TODO(joyeecheung): we are not a window - work with the upstream to
+    // add a new scope for us.
+    sandbox.document = {};  // Pretend we are Window
+    return context;
+  }
+
+  resultCallback(filename, test) {
+    switch (test.status) {
+      case 1:
+        this.fail(filename, test, 'FAILURE');
+        break;
+      case 2:
+        this.fail(filename, test, 'TIMEOUT');
+        break;
+      case 3:
+        this.fail(filename, test, 'INCOMPLETE');
+        break;
+      default:
+        this.succeed(filename, test);
+    }
+  }
+
+  completionCallback(filename, tests, harnessStatus) {
+    if (harnessStatus.status === 2) {
+      assert.fail(`test harness timed out in ${filename}`);
+    }
+    this.inProgress.delete(filename);
+    this.tryFinish();
+  }
+
+  tryFinish() {
+    if (this.inProgress.size > 0) {
+      return;
+    }
+
+    this.reportResults();
+  }
+
+  reportResults() {
+    const unexpectedFailures = [];
+    for (const [filename, items] of this.results) {
+      const test = this.tests.get(filename);
+      let title = test.meta && test.meta.title;
+      title = title ? `${filename} : ${title}` : filename;
+      console.log(`---- ${title} ----`);
+      for (const item of items) {
+        switch (item.type) {
+          case FAILED: {
+            if (test.failReason) {
+              console.log(`[EXPECTED_FAILURE] ${item.test.name}`);
+            } else {
+              console.log(`[UNEXPECTED_FAILURE] ${item.test.name}`);
+              unexpectedFailures.push([title, filename, item]);
+            }
+            break;
+          }
+          case PASSED: {
+            console.log(`[PASSED] ${item.test.name}`);
+            break;
+          }
+          case SKIPPED: {
+            console.log(`[SKIPPED] ${item.reason}`);
+            break;
+          }
+        }
+      }
+    }
+
+    if (unexpectedFailures.length > 0) {
+      for (const [title, filename, item] of unexpectedFailures) {
+        console.log(`---- ${title} ----`);
+        console.log(`[${item.reason}] ${item.test.name}`);
+        console.log(item.test.message);
+        console.log(item.test.stack);
+        const command = `${process.execPath} ${process.execArgv}` +
+                        ` ${require.main.filename} ${filename}`;
+        console.log(`Command: ${command}\n`);
+      }
+      assert.fail(`${unexpectedFailures.length} unexpected failures found`);
+    }
+  }
+
+  addResult(filename, item) {
+    const result = this.results.get(filename);
+    if (result) {
+      result.push(item);
+    } else {
+      this.results.set(filename, [item]);
+    }
+  }
+
+  succeed(filename, test) {
+    this.addResult(filename, {
+      type: PASSED,
+      test
+    });
+  }
+
+  fail(filename, test, reason) {
+    this.addResult(filename, {
+      type: FAILED,
+      test,
+      reason
+    });
+  }
+
+  skip(filename, reason) {
+    this.addResult(filename, {
+      type: SKIPPED,
+      reason
+    });
+  }
+
+  getMeta(code) {
+    const matches = code.match(/\/\/ META: .+/g);
+    if (!matches) {
+      return {};
+    } else {
+      const result = {};
+      for (const match of matches) {
+        const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
+        const key = parts[1];
+        const value = parts[2];
+        if (key === 'script') {
+          if (result[key]) {
+            result[key].push(value);
+          } else {
+            result[key] = [value];
+          }
+        } else {
+          result[key] = value;
+        }
+      }
+      return result;
+    }
+  }
+
+  mergeScripts(meta, content) {
+    if (!meta.script) {
+      return content;
+    }
+
+    // only one script
+    let result = '';
+    for (const script of meta.script) {
+      result += this.resource.fetch(script, false);
+    }
+
+    return result + content;
+  }
+
+  buildQueue() {
+    const queue = [];
+    for (const test of this.tests.values()) {
+      const filename = test.filename;
+      if (test.skipReason) {
+        this.skip(filename, test.skipReason);
+        continue;
+      }
+
+      if (!common.hasIntl && test.requireIntl()) {
+        this.skip(filename, 'missing Intl');
+        continue;
+      }
+
+      queue.push(test);
+    }
+    return queue;
+  }
+}
+
+module.exports = {
+  harness: harnessMock,
+  WPTRunner
+};

test/parallel/test-whatwg-url-constructor.js

@@ -8,7 +8,7 @@ if (!common.hasIntl) {
 const fixtures = require('../common/fixtures');
 const { URL, URLSearchParams } = require('url');
 const { test, assert_equals, assert_true, assert_throws } =
-  require('../common/wpt');
+  require('../common/wpt').harness;
 
 const request = {
   response: require(

test/parallel/test-whatwg-url-custom-searchparams-sort.js

@@ -4,9 +4,14 @@
 
 require('../common');
 const { URL, URLSearchParams } = require('url');
-const { test, assert_array_equals } = require('../common/wpt');
+const { test, assert_array_equals } = require('../common/wpt').harness;
 
-// Test bottom-up iterative stable merge sort
+// TODO(joyeecheung): upstream this to WPT, if possible - even
+// just as a test for large inputs. Other implementations may
+// have a similar cutoff anyway.
+
+// Test bottom-up iterative stable merge sort because we only use that
+// algorithm to sort > 100 search params.
 const tests = [{ input: '', output: [] }];
 const pairs = [];
 for (let i = 10; i < 100; i++) {

test/parallel/test-whatwg-url-custom-setters.js

@@ -10,9 +10,10 @@ if (!common.hasIntl) {
 
 const assert = require('assert');
 const URL = require('url').URL;
-const { test, assert_equals } = require('../common/wpt');
+const { test, assert_equals } = require('../common/wpt').harness;
 const fixtures = require('../common/fixtures');
 
+// TODO(joyeecheung): we should submit these to the upstream
 const additionalTestCases =
   require(fixtures.path('url-setter-tests-additional.js'));
 

test/parallel/test-whatwg-url-origin.js

@@ -7,7 +7,7 @@ if (!common.hasIntl) {
 
 const fixtures = require('../common/fixtures');
 const URL = require('url').URL;
-const { test, assert_equals } = require('../common/wpt');
+const { test, assert_equals } = require('../common/wpt').harness;
 
 const request = {
   response: require(

test/parallel/test-whatwg-url-setters.js

@@ -7,7 +7,7 @@ if (!common.hasIntl) {
 }
 
 const URL = require('url').URL;
-const { test, assert_equals } = require('../common/wpt');
+const { test, assert_equals } = require('../common/wpt').harness;
 const fixtures = require('../common/fixtures');
 
 const request = {

test/parallel/test-whatwg-url-toascii.js

@@ -7,7 +7,7 @@ if (!common.hasIntl) {
 
 const fixtures = require('../common/fixtures');
 const { URL } = require('url');
-const { test, assert_equals, assert_throws } = require('../common/wpt');
+const { test, assert_equals, assert_throws } = require('../common/wpt').harness;
 
 const request = {
   response: require(

test/wpt/README.md

@@ -0,0 +1,171 @@
+# Web Platform Tests
+
+The tests here are drivers for running the [Web Platform Tests][].
+
+See [`test/fixtures/wpt/README.md`][] for a hash of the last
+updated WPT commit for each module being covered here.
+
+See the json files in [the `status` folder](./status) for prerequisites,
+expected failures, and support status for specific tests in each module.
+
+Currently there are still some Web Platform Tests titled `test-whatwg-*`
+under `test/parallel` that have not been migrated to be run with the
+WPT harness and have automatic updates. There are also a few
+`test-whatwg-*-custom-*` tests that may need to be upstreamed.
+This folder covers the tests that have been migrated.
+
+<a id="add-tests"></a>
+## How to add tests for a new module
+
+### 1. Create a status file
+
+For example, to add the URL tests, add a `test/wpt/status/url.json` file.
+
+In the beginning, it's fine to leave an empty object `{}` in the file if
+it's not yet clear how compliant the implementation is,
+the requirements and expected failures can be figured out in a later step
+when the tests are run for the first time.
+
+See [Format of a status JSON file](#status-format) for details.
+
+### 2. Pull the WPT files
+
+Use the [git node wpt][] command to download the WPT files into
+`test/fixtures/wpt`. For example, to add URL tests:
+
+```text
+$ cd /path/to/node/project
+$ git node wpt url
+```
+
+### 3. Create the test driver
+
+For example, for the URL tests, add a file `test/wpt/test-whatwg-url.js`:
+
+```js
+'use strict';
+
+// This flag is required by the WPT Runner to patch the internals
+// for the tests to run in a vm.
+// Flags: --expose-internals
+
+require('../common');
+const { WPTRunner } = require('../common/wpt');
+
+const runner = new WPTRunner('url');
+
+// Copy global descriptors from the global object
+runner.copyGlobalsFromObject(global, ['URL', 'URLSearchParams']);
+// Define any additional globals with descriptors
+runner.defineGlobal('DOMException', {
+  get() {
+    return require('internal/domexception');
+  }
+});
+
+runner.runJsTests();
+```
+
+This driver is capable of running the tests located in `test/fixtures/wpt/url`
+with the WPT harness while taking the status file into account.
+
+### 4. Run the tests
+
+Run the test using `tools/test.py` and see if there are any failures.
+For example, to run all the URL tests under `test/fixtures/wpt/url`:
+
+```text
+$ tools/test.py wpt/test-whatwg-url
+```
+
+To run a specific test in WPT, for example, `url/url-searchparams.any.js`,
+pass the file name as argument to the corresponding test driver:
+
+```text
+node --expose-internals test/wpt/test-whatwg-url.js url-searchparams.any.js
+```
+
+If there are any failures, update the corresponding status file
+(in this case, `test/wpt/status/url.json`) to make the test pass.
+
+For example, to mark `url/url-searchparams.any.js` as expected to fail,
+add this to `test/wpt/status/url.json`:
+
+```json
+  "url-searchparams.any.js": {
+    "fail": "explain why the test fails, ideally with links"
+  }
+```
+
+See [Format of a status JSON file](#status-format) for details.
+
+### 5. Commit the changes and submit a Pull Request
+
+See [the contributing guide](../../CONTRIBUTING.md).
+
+## How to update tests for a module
+
+The tests can be updated in a way similar to how they are added.
+Run Step 2 and Step 4 of [adding tests for a new module](#add-tests).
+
+The [git node wpt][] command maintains the status of the local
+WPT subset, if no files are updated after running it for a module,
+the local subset is up to date and there is no need to update them
+until they are changed in the upstream.
+
+## How it works
+
+Note: currently this test suite only supports `.js` tests. There is
+ongoing work in the upstream to properly split out the tests into files
+that can be run in a shell environment like Node.js.
+
+### Getting the original test files and harness from WPT
+
+The original files and harness from WPT are downloaded and stored in
+`test/fixtures/wpt`.
+
+The [git node wpt][] command automate this process while maintaining a map
+containing the hash of the last updated commit for each module in
+`test/fixtures/wpt/versions.json` and [`test/fixtures/wpt/README.md`][].
+It also maintains the LICENSE file in `test/fixtures/wpt`.
+
+### Loading and running the tests
+
+Given a module, the `WPTRunner` class in [`test/common/wpt`](../common/wpt.js)
+loads:
+
+- `.js` test files (for example, `test/common/wpt/url/*.js` for `url`)
+- Status file (for example, `test/wpt/status/url.json` for `url`)
+- The WPT harness
+
+Then, for each test, it creates a vm with the globals and mocks,
+sets up the harness result hooks, loads the metadata in the test (including
+loading extra resources), and runs all the tests in that vm,
+skipping tests that cannot be run because of lack of dependency or
+expected failures.
+
+<a id="status-format"></a>
+## Format of a status JSON file
+
+```text
+{
+  "something.scope.js": {  // the file name
+    // Optional: If the requirement is not met, this test will be skipped
+    "requires": ["intl"],  // currently only intl is supported
+
+    // Optional: the test will be skipped with the reason printed
+    "skip": "explain why we cannot run a test that's supposed to pass",
+
+    // Optional: the test will be skipped with the reason printed
+    "fail": "explain why we the test is expected to fail"
+  }
+}
+```
+
+A test may have to be skipped because it depends on another irrelevant
+Web API, or certain harness has not been ported in our test runner yet.
+In that case it needs to be marked with `skip` instead of `fail`.
+
+[Web Platform Tests]: https://github.com/web-platform-tests/wpt
+[git node wpt]: https://github.com/nodejs/node-core-utils/blob/master/docs/git-node.md#git-node-wpt
+[`test/fixtures/wpt/README.md`]: ../fixtures/wpt/README.md

test/wpt/status/console.json

@@ -0,0 +1,5 @@
+{
+  "idlharness.any.js": {
+    "fail": ".table, .dir and .timeLog parameter lengths are wrong"
+  }
+}

test/wpt/status/url.json

@@ -0,0 +1,15 @@
+{
+  "toascii.window.js": {
+    "requires": ["intl"],
+    "skip": "TODO: port from .window.js"
+  },
+  "historical.any.js": {
+    "requires": ["intl"]
+  },
+  "urlencoded-parser.any.js": {
+    "fail": "missing Request and Response"
+  },
+  "idlharness.any.js": {
+    "fail": "getter/setter names are wrong, etc."
+  }
+}

test/wpt/test-whatwg-console.js

@@ -0,0 +1,13 @@
+'use strict';
+
+// Flags: --expose-internals
+
+require('../common');
+const { WPTRunner } = require('../common/wpt');
+
+const runner = new WPTRunner('console');
+
+// Copy global descriptors from the global object
+runner.copyGlobalsFromObject(global, ['console']);
+
+runner.runJsTests();

test/wpt/test-whatwg-url.js

@@ -0,0 +1,19 @@
+'use strict';
+
+// Flags: --expose-internals
+
+require('../common');
+const { WPTRunner } = require('../common/wpt');
+
+const runner = new WPTRunner('url');
+
+// Copy global descriptors from the global object
+runner.copyGlobalsFromObject(global, ['URL', 'URLSearchParams']);
+// Needed by urlsearchparams-constructor.any.js
+runner.defineGlobal('DOMException', {
+  get() {
+    return require('internal/domexception');
+  }
+});
+
+runner.runJsTests();

test/wpt/testcfg.py

@@ -0,0 +1,6 @@
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import testpy
+
+def GetConfiguration(context, root):
+  return testpy.ParallelTestConfiguration(context, root, 'wpt')

test/wpt/wpt.status

@@ -0,0 +1,21 @@
+prefix wpt
+
+# To mark a test as flaky, list the test name in the appropriate section
+# below, without ".js", followed by ": PASS,FLAKY". Example:
+# sample-test                        : PASS,FLAKY
+
+[true] # This section applies to all platforms
+
+[$system==win32]
+
+[$system==linux]
+
+[$system==macos]
+
+[$arch==arm || $arch==arm64]
+
+[$system==solaris] # Also applies to SmartOS
+
+[$system==freebsd]
+
+[$system==aix]