diff --git a/.vscode/settings.json b/.vscode/settings.json
index f79841a43..41bad73a9 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -42,6 +42,9 @@
     "LANG": "en-US",
     "TZ": "UTC"
   },
+
+  // These options are used by the `jestrunner.debug` command.
+  // They are not used by the `jestrunner.run` command.
   "jestrunner.debugOptions": {
     // Uncomment to debug integration tests
     // "attachSimplePort": 9223,
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
index 2984c675e..0df450d9c 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
@@ -19,8 +19,8 @@ import { DatabaseItem, DatabaseManager } from "../../../src/databases";
 import { CodeQLExtensionInterface } from "../../../src/extension";
 import { cleanDatabases, dbLoc, storagePath } from "./global.helper";
 import { importArchiveDatabase } from "../../../src/databaseFetcher";
-import { CodeQLCliServer } from "../../../src/cli";
-import { describeWithCodeQL, skipIfTrue } from "../cli";
+import { CliVersionConstraint, CodeQLCliServer } from "../../../src/cli";
+import { describeWithCodeQL } from "../cli";
 import { tmpDir } from "../../../src/helpers";
 import { createInitialQueryInfo } from "../../../src/run-queries-shared";
 import { QueryRunner } from "../../../src/queryRunner";
@@ -99,8 +99,6 @@ describeWithCodeQL()("Queries", () => {
   });
 
   describe("extension packs", () => {
-    skipIfTrue(qs.cliServer.cliConstraints.supportsQlpacksKind);
-
     const queryUsingExtensionPath = join(
       __dirname,
       "../..",
@@ -110,17 +108,38 @@ describeWithCodeQL()("Queries", () => {
     );
 
     it("should run a query that has an extension without looking for extensions in the workspace", async () => {
+      if (!(await supportsExtensionPacks())) {
+        console.log(
+          `Skipping test because it is only supported for CodeQL CLI versions < ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND}`,
+        );
+        return;
+      }
+
       await cli.setUseExtensionPacks(false);
       const parsedResults = await runQueryWithExtensions();
       expect(parsedResults).toEqual([1]);
     });
 
     it("should run a query that has an extension and look for extensions in the workspace", async () => {
+      if (!(await supportsExtensionPacks())) {
+        return;
+      }
+
       await cli.setUseExtensionPacks(true);
       const parsedResults = await runQueryWithExtensions();
       expect(parsedResults).toEqual([1, 2, 3, 4]);
     });
 
+    async function supportsExtensionPacks(): Promise<boolean> {
+      if (await qs.cliServer.cliConstraints.supportsQlpacksKind()) {
+        return true;
+      }
+      console.log(
+        `Skipping test because it is only supported for CodeQL CLI versions < ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND}`,
+      );
+      return false;
+    }
+
     async function runQueryWithExtensions() {
       const result = new CompletedQueryInfo(
         await qs.compileAndRunQueryAgainstDatabase(
diff --git a/extensions/ql-vscode/test/vscode-tests/cli.ts b/extensions/ql-vscode/test/vscode-tests/cli.ts
index e3bbe4653..e875c44ea 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli.ts
@@ -42,11 +42,3 @@ export function itWithCodeQL() {
 
   return it;
 }
-
-export function skipIfTrue(condition: () => Thenable<boolean>) {
-  return beforeEach(async () => {
-    if (await condition) {
-      pending();
-    }
-  });
-}