diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts
index fa848a9b7..e3b1b370b 100644
--- a/extensions/ql-vscode/src/config.ts
+++ b/extensions/ql-vscode/src/config.ts
@@ -119,14 +119,6 @@ export interface DistributionConfig {
   ownerName?: string;
   repositoryName?: string;
   onDidChangeConfiguration?: Event<void>;
-
-  /**
-   * This forces an update of the distribution, even if the settings haven't changed.
-   *
-   * This should only be used when the distribution has been updated outside of the extension
-   * and only in tests. It should not be called in production code.
-   */
-  forceUpdateConfiguration(): void;
 }
 
 // Query server configuration
@@ -265,7 +257,10 @@ export class DistributionConfigListener
   implements DistributionConfig
 {
   public get customCodeQlPath(): string | undefined {
-    return CUSTOM_CODEQL_PATH_SETTING.getValue() || undefined;
+    const testCliPath =
+      isIntegrationTestMode() &&
+      process.env.VSCODE_CODEQL_TESTING_CODEQL_CLI_TEST_PATH;
+    return CUSTOM_CODEQL_PATH_SETTING.getValue() || testCliPath || undefined;
   }
 
   public get includePrerelease(): boolean {
@@ -283,10 +278,6 @@ export class DistributionConfigListener
     );
   }
 
-  public forceUpdateConfiguration() {
-    this._onDidChangeConfiguration.fire(undefined);
-  }
-
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(
       DISTRIBUTION_CHANGE_SETTINGS,
diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js
index 9f95a10f0..0ec6d2585 100644
--- a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js
+++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js
@@ -22,6 +22,7 @@ const config = {
   extensionTestsEnv: {
     ...baseConfig.extensionTestsEnv,
     INTEGRATION_TEST_MODE: "true",
+    VSCODE_CODEQL_TESTING_CODEQL_CLI_TEST_PATH: process.env.CLI_PATH,
   },
   retries: 3,
 };
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-vscode.config.js b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-vscode.config.js
index 889d7c967..921418035 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-vscode.config.js
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-vscode.config.js
@@ -35,6 +35,7 @@ const config = {
   extensionTestsEnv: {
     ...baseConfig.extensionTestsEnv,
     INTEGRATION_TEST_MODE: "true",
+    VSCODE_CODEQL_TESTING_CODEQL_CLI_TEST_PATH: process.env.CLI_PATH,
   },
   retries: 3,
 };
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts
index 7b5b43833..4fcbfe775 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts
@@ -7,7 +7,7 @@ import {
 } from "../jest.activated-extension.setup";
 import { createWriteStream, existsSync, mkdirpSync } from "fs-extra";
 import { dirname } from "path";
-import { DB_URL, dbLoc, getActivatedExtension } from "../global.helper";
+import { DB_URL, dbLoc } from "../global.helper";
 import fetch from "node-fetch";
 
 beforeAll(async () => {
@@ -31,21 +31,6 @@ beforeAll(async () => {
   }
 
   await beforeAllAction();
-
-  // Activate the extension
-  const extension = await getActivatedExtension();
-
-  if (process.env.CLI_VERSION && process.env.CLI_VERSION !== "nightly") {
-    const cliVersion = await extension.cliServer.getVersion();
-
-    if (cliVersion.compare(process.env.CLI_VERSION) !== 0) {
-      // It seems like the CUSTOM_CODEQL_PATH_SETTING.updateValue() call in
-      // `beforeAllAction` doesn't fire the event that the config has changed.
-      // `forceUpdateConfiguration` will fire the event manually.
-      // This is a hacky workaround.
-      extension.distributionManager.config.forceUpdateConfiguration();
-    }
-  }
 });
 
 beforeEach(async () => {
diff --git a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts
index 3c20ad5a4..1879e0b5c 100644
--- a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts
+++ b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts
@@ -1,5 +1,4 @@
-import { CUSTOM_CODEQL_PATH_SETTING } from "../../src/config";
-import { ConfigurationTarget, env } from "vscode";
+import { env } from "vscode";
 import { beforeEachAction as testConfigBeforeEachAction } from "./test-config";
 import * as tmp from "tmp";
 import { realpathSync } from "fs-extra";
@@ -19,13 +18,6 @@ if (process.env.CI) {
 let removeStorage: tmp.DirResult["removeCallback"] | undefined;
 
 export async function beforeAllAction() {
-  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
-  await testConfigBeforeEachAction();
-  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
-    process.env.CLI_PATH,
-    ConfigurationTarget.Workspace,
-  );
-
   // Create the temp directory to be used as extension local storage.
   const dir = tmp.dirSync();
   let storagePath = realpathSync(dir.name);
@@ -46,11 +38,6 @@ export async function beforeEachAction() {
   jest.spyOn(env, "openExternal").mockResolvedValue(false);
 
   await testConfigBeforeEachAction();
-
-  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
-    process.env.CLI_PATH,
-    ConfigurationTarget.Workspace,
-  );
 }
 
 export async function afterAllAction() {