Add --retry support for end2end tests

package.json

@@ -21,14 +21,14 @@
         "pretty:check": "prettier --check ./",
         "pretty:fix": "prettier --write ./",
         "format": "npm run pretty:fix && npm run lint:fix",
-        "test:chrome": "node tests/run-unittests.mjs --browser chrome",
-        "test:firefox": "node tests/run-unittests.mjs --browser firefox",
-        "test:safari": "node tests/run-unittests.mjs --browser safari",
-        "test:edge": "node tests/run-unittests.mjs --browser edge",
-        "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome",
-        "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox",
-        "test-e2e:safari": "node tests/run-end2end.mjs --browser safari",
-        "test-e2e:edge": "node tests/run-end2end.mjs --browser edge"
+        "test:chrome": "node tests/run-unittests.mjs --browser=chrome",
+        "test:firefox": "node tests/run-unittests.mjs --browser=firefox",
+        "test:safari": "node tests/run-unittests.mjs --browser=safari",
+        "test:edge": "node tests/run-unittests.mjs --browser=edge",
+        "test-e2e:chrome": "node tests/run-end2end.mjs --browser=chrome",
+        "test-e2e:firefox": "node tests/run-end2end.mjs --browser=firefox",
+        "test-e2e:safari": "node tests/run-end2end.mjs --browser=safari",
+        "test-e2e:edge": "node tests/run-end2end.mjs --browser=edge"
     },
     "devDependencies": {
         "@babel/core": "^7.21.3",

tests/helper.mjs

@@ -4,9 +4,12 @@ import serve from "./server.mjs";
 
 import { Builder, Capabilities, logging } from "selenium-webdriver";
 
+export const DEFAULT_RETRIES = 1;
+
 const optionDefinitions = [
     { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." },
     { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." },
+    { name: "retry", type: Number, defaultValue: DEFAULT_RETRIES, description: "Number of retries for the tests on failure." },
     { name: "help", alias: "h", description: "Print this help text." },
 ];
 
@@ -38,6 +41,9 @@ export default async function testSetup(helpText) {
     if (!BROWSER)
         printHelp("No browser specified, use $BROWSER or --browser", 1);
 
+    if (options.retry < 0)
+        printHelp("Number of retries cannot be negative", 1);
+
     let capabilities;
     switch (BROWSER) {
         case "safari":
@@ -86,5 +92,5 @@ export default async function testSetup(helpText) {
         if (driver)
             driver.close();
     }
-    return { driver, PORT, stop };
+    return { driver, PORT, stop, retry: options.retry };
 }

tests/run-end2end.mjs

@@ -1,15 +1,15 @@
 #! /usr/bin/env node
 
 import assert from "assert";
-import testSetup from "./helper.mjs";
+import testSetup, { DEFAULT_RETRIES } from "./helper.mjs";
 import { benchmarkConfigurator } from "../resources/benchmark-configurator.mjs";
 
 const HELP = `
 This script runs end2end tests by invoking the benchmark via the main
 Speedometer page in /index.html.
 `.trim();
 
-const { driver, PORT, stop } = await testSetup(HELP);
+const { driver, PORT, stop, retry } = await testSetup(HELP);
 
 const suites = benchmarkConfigurator.suites;
 
@@ -112,17 +112,34 @@ async function testDeveloperMode() {
 
 async function test() {
     try {
-        await driver.manage().setTimeouts({ script: 60000 });
-        await testIterations();
-        await testAll();
-        await testDeveloperMode();
-        console.log("\nTests complete!");
-    } catch (e) {
-        console.error("\nTests failed!");
-        throw e;
+        await testWithRetries(retry);
     } finally {
         stop();
     }
 }
 
+async function testWithRetries(retries = DEFAULT_RETRIES) {
+    const maxAttempts = retries + 1;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            await tryTests();
+            return;
+        } catch (e) {
+            console.error(`\nTests failed on attempt ${attempt}!`);
+            if (attempt === maxAttempts)
+                throw e;
+
+            console.log(`Retrying... (${attempt}/${maxAttempts})`);
+        }
+    }
+}
+
+async function tryTests() {
+    await driver.manage().setTimeouts({ script: 60000 });
+    await testIterations();
+    await testAll();
+    await testDeveloperMode();
+    console.log("\nTests complete!");
+}
+
 setImmediate(test);