From 3a1219bb645daae569cb7e736e812fc471fd0a13 Mon Sep 17 00:00:00 2001
From: Jason Reed <jcreed@semmle.com>
Date: Thu, 5 Mar 2020 09:26:26 -0500
Subject: [PATCH 1/2] Retry integration tests when they fail

---
 .../src/vscode-tests/run-integration-tests.ts | 40 +++++++++++++++++--
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts b/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
index 03c84d009..2d292e0a8 100644
--- a/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
+++ b/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
@@ -1,6 +1,41 @@
 import * as path from 'path';
 import { runTests } from 'vscode-test';
 
+// A subset of the fields in TestOptions from vscode-test, which we
+// would simply use instead, but for the fact that it doesn't export
+// it.
+type Suite = {
+  extensionDevelopmentPath: string,
+  extensionTestsPath: string,
+  launchArgs: string[]
+};
+
+/**
+ * Run an integration test suite `suite` at most `tries` times, or
+ * until it succeeds, whichever comes first.
+ *
+ * TODO: Presently there is no way to distinguish a legitimately
+ * failed test run from the test runner being terminated by a signal.
+ * If in the future there arises a way to distinguish these cases
+ * (e.g. https://github.com/microsoft/vscode-test/pull/56) only retry
+ * in the terminated-by-signal case.
+ */
+async function runTestsWithRetry(suite: Suite, tries: number): Promise<void> {
+  for (let t = 0; t < tries; t++) {
+    try {
+      // Download and unzip VS Code if necessary, and run the integration test suite.
+      await runTests(suite);
+      return;
+    } catch (err) {
+      console.error(`Exception raised while running tests: ${err}`);
+      if (t < tries - 1)
+        console.error('Retrying...');
+    }
+  }
+  console.error(`Tried running suite ${tries} time(s), still failed, giving up.`);
+  process.exit(1);
+}
+
 /**
  * Integration test runner. Launches the VSCode Extension Development Host with this extension installed.
  * See https://github.com/microsoft/vscode-test/blob/master/sample/test/runTest.ts
@@ -32,11 +67,10 @@ async function main() {
     ];
 
     for (const integrationTestSuite of integrationTestSuites) {
-      // Download and unzip VS Code if necessary, and run the integration test suite.
-      await runTests(integrationTestSuite);
+      await runTestsWithRetry(integrationTestSuite, 2);
     }
   } catch (err) {
-    console.error('Failed to run tests');
+    console.error('Unexpected exception while running tests');
     process.exit(1);
   }
 }

From 87e563e24e77ecb134df540f9c235580982b13f2 Mon Sep 17 00:00:00 2001
From: Jason Reed <jcreed@semmle.com>
Date: Thu, 5 Mar 2020 12:47:02 -0500
Subject: [PATCH 2/2] Review comments.

---
 .../ql-vscode/src/vscode-tests/run-integration-tests.ts     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts b/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
index 2d292e0a8..b54483d7e 100644
--- a/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
+++ b/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
@@ -29,7 +29,7 @@ async function runTestsWithRetry(suite: Suite, tries: number): Promise<void> {
     } catch (err) {
       console.error(`Exception raised while running tests: ${err}`);
       if (t < tries - 1)
-        console.error('Retrying...');
+        console.log('Retrying...');
     }
   }
   console.error(`Tried running suite ${tries} time(s), still failed, giving up.`);
@@ -67,10 +67,10 @@ async function main() {
     ];
 
     for (const integrationTestSuite of integrationTestSuites) {
-      await runTestsWithRetry(integrationTestSuite, 2);
+      await runTestsWithRetry(integrationTestSuite, 3);
     }
   } catch (err) {
-    console.error('Unexpected exception while running tests');
+    console.error(`Unexpected exception while running tests: ${err}`);
     process.exit(1);
   }
 }