Merge pull request #3233 from github/dbartol/mrva-multi-bundle

Use CLI to bundle packs for MRVA

extensions/ql-vscode/src/codeql-cli/cli-command.ts

@@ -3,7 +3,10 @@ import { promisify } from "util";
 
 import type { BaseLogger } from "../common/logging";
 import type { ProgressReporter } from "../common/logging/vscode";
-import { getChildProcessErrorMessage } from "../common/helpers-pure";
+import {
+  getChildProcessErrorMessage,
+  getErrorMessage,
+} from "../common/helpers-pure";
 
 /**
  * Flags to pass to all cli commands.
@@ -11,26 +14,27 @@ import { getChildProcessErrorMessage } from "../common/helpers-pure";
 export const LOGGING_FLAGS = ["-v", "--log-to-stderr"];
 
 /**
- * Runs a CodeQL CLI command without invoking the CLI server, returning the output as a string.
+ * Runs a CodeQL CLI command without invoking the CLI server, deserializing the output as JSON.
  * @param codeQlPath The path to the CLI.
  * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
  * @param commandArgs The arguments to pass to the `codeql` command.
  * @param description Description of the action being run, to be shown in log and error messages.
  * @param logger Logger to write command log messages, e.g. to an output channel.
  * @param progressReporter Used to output progress messages, e.g. to the status bar.
- * @returns The contents of the command's stdout, if the command succeeded.
+ * @returns A JSON object parsed from the contents of the command's stdout, if the command succeeded.
  */
-export async function runCodeQlCliCommand(
+export async function runJsonCodeQlCliCommand<OutputType>(
   codeQlPath: string,
   command: string[],
   commandArgs: string[],
   description: string,
   logger: BaseLogger,
   progressReporter?: ProgressReporter,
-): Promise<string> {
+): Promise<OutputType> {
   // Add logging arguments first, in case commandArgs contains positional parameters.
   const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
   const argsString = args.join(" ");
+  let stdout: string;
   try {
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
@@ -41,10 +45,18 @@ export async function runCodeQlCliCommand(
     const result = await promisify(execFile)(codeQlPath, args);
     void logger.log(result.stderr);
     void logger.log("CLI command succeeded.");
-    return result.stdout;
+    stdout = result.stdout;
   } catch (err) {
     throw new Error(
       `${description} failed: ${getChildProcessErrorMessage(err)}`,
     );
   }
+
+  try {
+    return JSON.parse(stdout) as OutputType;
+  } catch (err) {
+    throw new Error(
+      `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,25 +1,49 @@
 import type { SemVer } from "semver";
 import { parse } from "semver";
-import { runCodeQlCliCommand } from "./cli-command";
+import { runJsonCodeQlCliCommand } from "./cli-command";
 import type { Logger } from "../common/logging";
 import { getErrorMessage } from "../common/helpers-pure";
 
+interface VersionResult {
+  version: string;
+  features: CliFeatures | undefined;
+}
+
+export interface CliFeatures {
+  featuresInVersionResult?: boolean;
+  mrvaPackCreate?: boolean;
+}
+
+export interface VersionAndFeatures {
+  version: SemVer;
+  features: CliFeatures;
+}
+
 /**
  * Get the version of a CodeQL CLI.
  */
 export async function getCodeQlCliVersion(
   codeQlPath: string,
   logger: Logger,
-): Promise<SemVer | undefined> {
+): Promise<VersionAndFeatures | undefined> {
   try {
-    const output: string = await runCodeQlCliCommand(
+    const output: VersionResult = await runJsonCodeQlCliCommand<VersionResult>(
       codeQlPath,
       ["version"],
-      ["--format=terse"],
+      ["--format=json"],
       "Checking CodeQL version",
       logger,
     );
-    return parse(output.trim()) || undefined;
+
+    const version = parse(output.version.trim()) || undefined;
+    if (version === undefined) {
+      return undefined;
+    }
+
+    return {
+      version,
+      features: output.features ?? {},
+    };
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -34,6 +34,7 @@ import { QueryLanguage } from "../common/query-language";
 import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
 import type { Position } from "../query-server/messages";
 import { LOGGING_FLAGS } from "./cli-command";
+import type { CliFeatures, VersionAndFeatures } from "./cli-version";
 
 /**
  * The version of the SARIF format that we are using.
@@ -203,7 +204,9 @@ type OnLineCallback = (
   line: string,
 ) => Promise<string | undefined> | string | undefined;
 
-type VersionChangedListener = (newVersion: SemVer | undefined) => void;
+type VersionChangedListener = (
+  newVersionAndFeatures: VersionAndFeatures | undefined,
+) => void;
 
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
@@ -221,8 +224,8 @@ export class CodeQLCliServer implements Disposable {
   /**  A buffer with a single null byte. */
   nullBuffer: Buffer;
 
-  /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: SemVer | undefined;
+  /** Version of current cli and its supported features, lazily computed by the `getVersion()` method */
+  private _versionAndFeatures: VersionAndFeatures | undefined;
 
   private _versionChangedListeners: VersionChangedListener[] = [];
 
@@ -298,7 +301,7 @@ export class CodeQLCliServer implements Disposable {
     const callback = (): void => {
       try {
         this.killProcessIfRunning();
-        this._version = undefined;
+        this._versionAndFeatures = undefined;
         this._supportedLanguages = undefined;
       } finally {
         this.runNext();
@@ -1427,16 +1430,28 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Compile a CodeQL pack and bundle it into a single file.
+   *
+   * @param sourcePackDir The directory of the input CodeQL pack.
+   * @param workspaceFolders The workspace folders to search for additional packs.
+   * @param outputBundleFile The path to the output bundle file.
+   * @param outputPackDir The directory to contain the unbundled output pack.
+   * @param moreOptions Additional options to be passed to `codeql pack bundle`.
+   */
   async packBundle(
-    dir: string,
+    sourcePackDir: string,
     workspaceFolders: string[],
-    outputPath: string,
+    outputBundleFile: string,
+    outputPackDir: string,
     moreOptions: string[],
   ): Promise<void> {
     const args = [
       "-o",
-      outputPath,
-      dir,
+      outputBundleFile,
+      sourcePackDir,
+      "--pack-path",
+      outputPackDir,
       ...moreOptions,
       ...this.getAdditionalPacksArg(workspaceFolders),
     ];
@@ -1491,27 +1506,35 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  public async getVersion() {
-    if (!this._version) {
+  public async getVersion(): Promise<SemVer> {
+    return (await this.getVersionAndFeatures()).version;
+  }
+
+  public async getFeatures(): Promise<CliFeatures> {
+    return (await this.getVersionAndFeatures()).features;
+  }
+
+  private async getVersionAndFeatures(): Promise<VersionAndFeatures> {
+    if (!this._versionAndFeatures) {
       try {
-        const newVersion = await this.refreshVersion();
-        this._version = newVersion;
+        const newVersionAndFeatures = await this.refreshVersion();
+        this._versionAndFeatures = newVersionAndFeatures;
         this._versionChangedListeners.forEach((listener) =>
-          listener(newVersion),
+          listener(newVersionAndFeatures),
         );
 
         // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
         await this.app.commands.execute(
           "setContext",
           "codeql.supportsQuickEvalCount",
-          newVersion.compare(
+          newVersionAndFeatures.version.compare(
             CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
           ) >= 0,
         );
         await this.app.commands.execute(
           "setContext",
           "codeql.supportsTrimCache",
-          newVersion.compare(
+          newVersionAndFeatures.version.compare(
             CliVersionConstraint.CLI_VERSION_WITH_TRIM_CACHE,
           ) >= 0,
         );
@@ -1522,23 +1545,23 @@ export class CodeQLCliServer implements Disposable {
         throw e;
       }
     }
-    return this._version;
+    return this._versionAndFeatures;
   }
 
   public addVersionChangedListener(listener: VersionChangedListener) {
-    if (this._version) {
-      listener(this._version);
+    if (this._versionAndFeatures) {
+      listener(this._versionAndFeatures);
     }
     this._versionChangedListeners.push(listener);
   }
 
-  private async refreshVersion() {
+  private async refreshVersion(): Promise<VersionAndFeatures> {
     const distribution = await this.distributionProvider.getDistribution();
     switch (distribution.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
       // eslint-disable-next-line no-fallthrough -- Intentional fallthrough
       case FindDistributionResultKind.IncompatibleDistribution:
-        return distribution.version;
+        return distribution.versionAndFeatures;
 
       default:
         // We should not get here because if no distributions are available, then
@@ -1755,4 +1778,8 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA,
     );
   }
+
+  async supportsMrvaPackCreate(): Promise<boolean> {
+    return (await this.cli.getFeatures()).mrvaPackCreate === true;
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -1,11 +1,11 @@
 import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
-import type { SemVer } from "semver";
 import { Range, satisfies } from "semver";
 import type { Event, ExtensionContext } from "vscode";
 import type { DistributionConfig } from "../config";
 import { extLogger } from "../common/logging/vscode";
+import type { VersionAndFeatures } from "./cli-version";
 import { getCodeQlCliVersion } from "./cli-version";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { reportStreamProgress } from "../common/vscode/progress";
@@ -88,11 +88,11 @@ export class DistributionManager implements DistributionProvider {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(
+    const versionAndFeatures = await getCodeQlCliVersion(
       distribution.codeQlPath,
       extLogger,
     );
-    if (version === undefined) {
+    if (versionAndFeatures === undefined) {
       return {
         distribution,
         kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
@@ -119,17 +119,21 @@ export class DistributionManager implements DistributionProvider {
       distribution.kind !== DistributionKind.ExtensionManaged ||
       this.config.includePrerelease;
 
-    if (!satisfies(version, this.versionRange, { includePrerelease })) {
+    if (
+      !satisfies(versionAndFeatures.version, this.versionRange, {
+        includePrerelease,
+      })
+    ) {
       return {
         distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
-        version,
+        versionAndFeatures,
       };
     }
     return {
       distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
-      version,
+      versionAndFeatures,
     };
   }
 
@@ -599,7 +603,7 @@ interface DistributionResult {
 
 interface CompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.CompatibleDistribution;
-  version: SemVer;
+  versionAndFeatures: VersionAndFeatures;
 }
 
 interface UnknownCompatibilityDistributionResult extends DistributionResult {
@@ -608,7 +612,7 @@ interface UnknownCompatibilityDistributionResult extends DistributionResult {
 
 interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: SemVer;
+  versionAndFeatures: VersionAndFeatures;
 }
 
 interface NoDistributionResult {

extensions/ql-vscode/src/common/short-paths.ts

@@ -0,0 +1,117 @@
+import { platform } from "os";
+import { basename, dirname, join, normalize, resolve } from "path";
+import { lstat, readdir } from "fs/promises";
+import type { BaseLogger } from "./logging";
+
+/**
+ * Expands a path that potentially contains 8.3 short names (e.g. "C:\PROGRA~1" instead of "C:\Program Files").
+ *
+ * See https://en.wikipedia.org/wiki/8.3_filename if you're not familiar with Windows 8.3 short names.
+ *
+ * @param shortPath The path to expand.
+ * @returns A normalized, absolute path, with any short components expanded.
+ */
+export async function expandShortPaths(
+  shortPath: string,
+  logger: BaseLogger,
+): Promise<string> {
+  const absoluteShortPath = normalize(resolve(shortPath));
+  if (platform() !== "win32") {
+    // POSIX doesn't have short paths.
+    return absoluteShortPath;
+  }
+
+  void logger.log(`Expanding short paths in: ${absoluteShortPath}`);
+  // A quick check to see if there might be any short components.
+  // There might be a case where a short component doesn't contain a `~`, but if there is, I haven't
+  // found it.
+  // This may find long components that happen to have a '~', but that's OK.
+  if (absoluteShortPath.indexOf("~") < 0) {
+    // No short components to expand.
+    void logger.log(`Skipping due to no short components`);
+    return absoluteShortPath;
+  }
+
+  return await expandShortPathRecursive(absoluteShortPath, logger);
+}
+
+/**
+ * Expand a single short path component
+ * @param dir The absolute path of the directory containing the short path component.
+ * @param shortBase The shot path component to expand.
+ * @returns The expanded path component.
+ */
+async function expandShortPathComponent(
+  dir: string,
+  shortBase: string,
+  logger: BaseLogger,
+): Promise<string> {
+  void logger.log(`Expanding short path component: ${shortBase}`);
+
+  const fullPath = join(dir, shortBase);
+
+  // Use `lstat` instead of `stat` to avoid following symlinks.
+  const stats = await lstat(fullPath, { bigint: true });
+  if (stats.dev === BigInt(0) || stats.ino === BigInt(0)) {
+    // No inode info, so we won't be able to find this in the directory listing.
+    void logger.log(`No inode info available. Skipping.`);
+    return shortBase;
+  }
+  void logger.log(`dev/inode: ${stats.dev}/${stats.ino}`);
+
+  try {
+    // Enumerate the children of the parent directory, and try to find one with the same dev/inode.
+    const children = await readdir(dir);
+    for (const child of children) {
+      void logger.log(`considering child: ${child}`);
+      try {
+        const childStats = await lstat(join(dir, child), { bigint: true });
+        void logger.log(`child dev/inode: ${childStats.dev}/${childStats.ino}`);
+        if (childStats.dev === stats.dev && childStats.ino === stats.ino) {
+          // Found a match.
+          void logger.log(`Found a match: ${child}`);
+          return child;
+        }
+      } catch (e) {
+        // Can't read stats for the child, so skip it.
+        void logger.log(`Error reading stats for child: ${e}`);
+      }
+    }
+  } catch (e) {
+    // Can't read the directory, so we won't be able to find this in the directory listing.
+    void logger.log(`Error reading directory: ${e}`);
+    return shortBase;
+  }
+
+  void logger.log(`No match found. Returning original.`);
+  return shortBase;
+}
+
+/**
+ * Expand the short path components in a path, including those in ancestor directories.
+ * @param shortPath The path to expand.
+ * @returns The expanded path.
+ */
+async function expandShortPathRecursive(
+  shortPath: string,
+  logger: BaseLogger,
+): Promise<string> {
+  const shortBase = basename(shortPath);
+  if (shortBase.length === 0) {
+    // We've reached the root.
+    return shortPath;
+  }
+
+  const dir = await expandShortPathRecursive(dirname(shortPath), logger);
+  void logger.log(`dir: ${dir}`);
+  void logger.log(`base: ${shortBase}`);
+  if (shortBase.indexOf("~") < 0) {
+    // This component doesn't have a short name, so just append it to the (long) parent.
+    void logger.log(`Component is not a short name`);
+    return join(dir, shortBase);
+  }
+
+  // This component looks like it has a short name, so try to expand it.
+  const longBase = await expandShortPathComponent(dir, shortBase, logger);
+  return join(dir, longBase);
+}

extensions/ql-vscode/src/extension.ts

@@ -430,7 +430,7 @@ export async function activate(
       codeQlExtension.variantAnalysisManager,
     );
     codeQlExtension.cliServer.addVersionChangedListener((ver) => {
-      telemetryListener.cliVersion = ver;
+      telemetryListener.cliVersion = ver?.version;
     });
 
     let unsupportedWarningShown = false;
@@ -443,13 +443,16 @@ export async function activate(
         return;
       }
 
-      if (CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION.compare(ver) < 0) {
+      if (
+        CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION.compare(ver.version) <
+        0
+      ) {
         return;
       }
 
       void showAndLogWarningMessage(
         extLogger,
-        `You are using an unsupported version of the CodeQL CLI (${ver}). ` +
+        `You are using an unsupported version of the CodeQL CLI (${ver.version}). ` +
           `The minimum supported version is ${CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION}. ` +
           `Please upgrade to a newer version of the CodeQL CLI.`,
       );
@@ -604,7 +607,7 @@ async function getDistributionDisplayingDistributionWarnings(
   switch (result.kind) {
     case FindDistributionResultKind.CompatibleDistribution:
       void extLogger.log(
-        `Found compatible version of CodeQL CLI (version ${result.version.raw})`,
+        `Found compatible version of CodeQL CLI (version ${result.versionAndFeatures.version.raw})`,
       );
       break;
     case FindDistributionResultKind.IncompatibleDistribution: {
@@ -624,7 +627,7 @@ async function getDistributionDisplayingDistributionWarnings(
 
       void showAndLogWarningMessage(
         extLogger,
-        `The current version of the CodeQL CLI (${result.version.raw}) ` +
+        `The current version of the CodeQL CLI (${result.versionAndFeatures.version.raw}) ` +
           `is incompatible with this extension. ${fixGuidanceMessage}`,
       );
       break;

extensions/ql-vscode/src/variant-analysis/run-remote-query.ts

@@ -3,6 +3,7 @@ import { Uri, window } from "vscode";
 import { relative, join, sep, dirname, parse, basename } from "path";
 import { dump, load } from "js-yaml";
 import { copy, writeFile, readFile, mkdirp } from "fs-extra";
+import type { DirectoryResult } from "tmp-promise";
 import { dir, tmpName } from "tmp-promise";
 import { tmpDir } from "../tmp-dir";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
@@ -37,6 +38,7 @@ import type { QueryLanguage } from "../common/query-language";
 import { tryGetQueryMetadata } from "../codeql-cli/query-metadata";
 import { askForLanguage, findLanguage } from "../codeql-cli/query-language";
 import type { QlPackFile } from "../packaging/qlpack-file";
+import { expandShortPaths } from "../common/short-paths";
 
 /**
  * Well-known names for the query pack used by the server.
@@ -58,21 +60,52 @@ interface GeneratedQueryPack {
 async function generateQueryPack(
   cliServer: CodeQLCliServer,
   queryFile: string,
-  queryPackDir: string,
+  tmpDir: RemoteQueryTempDir,
 ): Promise<GeneratedQueryPack> {
   const originalPackRoot = await findPackRoot(queryFile);
   const packRelativePath = relative(originalPackRoot, queryFile);
-  const targetQueryFileName = join(queryPackDir, packRelativePath);
   const workspaceFolders = getOnDiskWorkspaceFolders();
+  const extensionPacks = await getExtensionPacksToInject(
+    cliServer,
+    workspaceFolders,
+  );
 
-  let language: QueryLanguage | undefined;
+  const mustSynthesizePack =
+    (await getQlPackPath(originalPackRoot)) === undefined;
+  const cliSupportsMrvaPackCreate =
+    await cliServer.cliConstraints.supportsMrvaPackCreate();
 
-  // Check if the query is already in a query pack.
-  // If so, copy the entire query pack to the temporary directory.
-  // Otherwise, copy only the query file to the temporary directory
-  // and generate a synthetic query pack.
-  if (await getQlPackPath(originalPackRoot)) {
-    // don't include ql files. We only want the queryFile to be copied.
+  const language: QueryLanguage | undefined = mustSynthesizePack
+    ? await askForLanguage(cliServer) // open popup to ask for language if not already hardcoded
+    : await findLanguage(cliServer, Uri.file(queryFile));
+  if (!language) {
+    throw new UserCancellationException("Could not determine language");
+  }
+
+  let queryPackDir: string;
+  let needsInstall: boolean;
+  if (mustSynthesizePack) {
+    // This section applies whether or not the CLI supports MRVA pack creation directly.
+
+    queryPackDir = tmpDir.queryPackDir;
+
+    // Synthesize a query pack for the query.
+    // copy only the query file to the query pack directory
+    // and generate a synthetic query pack
+    await createNewQueryPack(
+      queryFile,
+      queryPackDir,
+      language,
+      packRelativePath,
+    );
+    // Clear the cliServer cache so that the previous qlpack text is purged from the CLI.
+    await cliServer.clearCache();
+
+    // Install packs, since we just synthesized a dependency on the language's standard library.
+    needsInstall = true;
+  } else if (!cliSupportsMrvaPackCreate) {
+    // We need to copy the query pack to a temporary directory and then fix it up to work with MRVA.
+    queryPackDir = tmpDir.queryPackDir;
     await copyExistingQueryPack(
       cliServer,
       originalPackRoot,
@@ -81,52 +114,55 @@ async function generateQueryPack(
       packRelativePath,
     );
 
-    language = await findLanguage(cliServer, Uri.file(targetQueryFileName));
+    // We should already have all the dependencies available, but these older versions of the CLI
+    // have a bug where they will not search `--additional-packs` during validation in `codeql pack bundle`.
+    // Installing the packs will ensure that any extension packs get put in the right place.
+    needsInstall = true;
   } else {
-    // open popup to ask for language if not already hardcoded
-    language = await askForLanguage(cliServer);
-
-    // copy only the query file to the query pack directory
-    // and generate a synthetic query pack
-    await createNewQueryPack(
-      queryFile,
-      queryPackDir,
-      targetQueryFileName,
-      language,
-      packRelativePath,
-    );
-  }
-  if (!language) {
-    throw new UserCancellationException("Could not determine language.");
+    // The CLI supports creating a MRVA query pack directly from the source pack.
+    queryPackDir = originalPackRoot;
+    // We expect any dependencies to be available already.
+    needsInstall = false;
   }
 
-  // Clear the cliServer cache so that the previous qlpack text is purged from the CLI.
-  await cliServer.clearCache();
+  if (needsInstall) {
+    // Install the dependencies of the synthesized query pack.
+    await cliServer.packInstall(queryPackDir, {
+      workspaceFolders,
+    });
 
-  let precompilationOpts: string[] = [];
-  if (await cliServer.cliConstraints.usesGlobalCompilationCache()) {
-    precompilationOpts = ["--qlx"];
-  } else {
-    const ccache = join(originalPackRoot, ".cache");
+    // Clear the CLI cache so that the most recent qlpack lock file is used.
+    await cliServer.clearCache();
+  }
+
+  let precompilationOpts: string[];
+  if (cliSupportsMrvaPackCreate) {
     precompilationOpts = [
-      "--qlx",
-      "--no-default-compilation-cache",
-      `--compilation-cache=${ccache}`,
+      "--mrva",
+      "--query",
+      join(queryPackDir, packRelativePath),
+      // We need to specify the extension packs as dependencies so that they are included in the MRVA pack.
+      // The version range doesn't matter, since they'll always be found by source lookup.
+      ...extensionPacks.map((p) => `--extension-pack=${p}@*`),
     ];
+  } else {
+    if (await cliServer.cliConstraints.usesGlobalCompilationCache()) {
+      precompilationOpts = ["--qlx"];
+    } else {
+      const cache = join(originalPackRoot, ".cache");
+      precompilationOpts = [
+        "--qlx",
+        "--no-default-compilation-cache",
+        `--compilation-cache=${cache}`,
+      ];
+    }
+
+    if (extensionPacks.length > 0) {
+      await addExtensionPacksAsDependencies(queryPackDir, extensionPacks);
+    }
   }
 
-  if (await cliServer.useExtensionPacks()) {
-    await injectExtensionPacks(cliServer, queryPackDir, workspaceFolders);
-  }
-
-  await cliServer.packInstall(queryPackDir, {
-    workspaceFolders,
-  });
-
-  // Clear the CLI cache so that the most recent qlpack lock file is used.
-  await cliServer.clearCache();
-
-  const bundlePath = await getPackedBundlePath(queryPackDir);
+  const bundlePath = tmpDir.bundleFile;
   void extLogger.log(
     `Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`,
   );
@@ -134,6 +170,7 @@ async function generateQueryPack(
     queryPackDir,
     workspaceFolders,
     bundlePath,
+    tmpDir.compiledPackDir,
     precompilationOpts,
   );
   const base64Pack = (await readFile(bundlePath)).toString("base64");
@@ -146,11 +183,11 @@ async function generateQueryPack(
 async function createNewQueryPack(
   queryFile: string,
   queryPackDir: string,
-  targetQueryFileName: string,
   language: string | undefined,
   packRelativePath: string,
 ) {
   void extLogger.log(`Copying ${queryFile} to ${queryPackDir}`);
+  const targetQueryFileName = join(queryPackDir, packRelativePath);
   await copy(queryFile, targetQueryFileName);
   void extLogger.log("Generating synthetic query pack");
   const syntheticQueryPack = {
@@ -242,19 +279,37 @@ function isFileSystemRoot(dir: string): boolean {
   return pathObj.root === dir && pathObj.base === "";
 }
 
-async function createRemoteQueriesTempDirectory() {
-  const remoteQueryDir = await dir({
+interface RemoteQueryTempDir {
+  remoteQueryDir: DirectoryResult;
+  queryPackDir: string;
+  compiledPackDir: string;
+  bundleFile: string;
+}
+
+async function createRemoteQueriesTempDirectory(): Promise<RemoteQueryTempDir> {
+  const shortRemoteQueryDir = await dir({
     dir: tmpDir.name,
     unsafeCleanup: true,
   });
+  // Expand 8.3 filenames here to work around a CLI bug where `codeql pack bundle` produces an empty
+  // archive if the pack path contains any 8.3 components.
+  const remoteQueryDir = {
+    ...shortRemoteQueryDir,
+    path: await expandShortPaths(shortRemoteQueryDir.path, extLogger),
+  };
   const queryPackDir = join(remoteQueryDir.path, "query-pack");
   await mkdirp(queryPackDir);
-  return { remoteQueryDir, queryPackDir };
+  const compiledPackDir = join(remoteQueryDir.path, "compiled-pack");
+  const bundleFile = await expandShortPaths(
+    await getPackedBundlePath(tmpDir.name),
+    extLogger,
+  );
+  return { remoteQueryDir, queryPackDir, compiledPackDir, bundleFile };
 }
 
-async function getPackedBundlePath(queryPackDir: string) {
+async function getPackedBundlePath(remoteQueryDir: string): Promise<string> {
   return tmpName({
-    dir: dirname(queryPackDir),
+    dir: remoteQueryDir,
     postfix: "generated.tgz",
     prefix: "qlpack",
   });
@@ -322,15 +377,14 @@ export async function prepareRemoteQueryRun(
     throw new UserCancellationException("Cancelled");
   }
 
-  const { remoteQueryDir, queryPackDir } =
-    await createRemoteQueriesTempDirectory();
+  const tempDir = await createRemoteQueriesTempDirectory();
 
   let pack: GeneratedQueryPack;
 
   try {
-    pack = await generateQueryPack(cliServer, queryFile, queryPackDir);
+    pack = await generateQueryPack(cliServer, queryFile, tempDir);
   } finally {
-    await remoteQueryDir.cleanup();
+    await tempDir.remoteQueryDir.cleanup();
   }
 
   const { base64Pack, language } = pack;
@@ -397,11 +451,38 @@ async function fixPackFile(
   await writeFile(packPath, dump(qlpack));
 }
 
-async function injectExtensionPacks(
+async function getExtensionPacksToInject(
   cliServer: CodeQLCliServer,
-  queryPackDir: string,
   workspaceFolders: string[],
-) {
+): Promise<string[]> {
+  const result: string[] = [];
+  if (await cliServer.useExtensionPacks()) {
+    const extensionPacks = await cliServer.resolveQlpacks(
+      workspaceFolders,
+      true,
+    );
+    Object.entries(extensionPacks).forEach(([name, paths]) => {
+      // We are guaranteed that there is at least one path found for each extension pack.
+      // If there are multiple paths, then we have a problem. This means that there is
+      // ambiguity in which path to use. This is an error.
+      if (paths.length > 1) {
+        throw new Error(
+          `Multiple versions of extension pack '${name}' found: ${paths.join(
+            ", ",
+          )}`,
+        );
+      }
+      result.push(name);
+    });
+  }
+
+  return result;
+}
+
+async function addExtensionPacksAsDependencies(
+  queryPackDir: string,
+  extensionPacks: string[],
+): Promise<void> {
   const qlpackFile = await getQlPackPath(queryPackDir);
   if (!qlpackFile) {
     throw new Error(
@@ -410,24 +491,13 @@ async function injectExtensionPacks(
       )} file in '${queryPackDir}'`,
     );
   }
+
   const syntheticQueryPack = load(
     await readFile(qlpackFile, "utf8"),
   ) as QlPackFile;
 
   const dependencies = syntheticQueryPack.dependencies ?? {};
-
-  const extensionPacks = await cliServer.resolveQlpacks(workspaceFolders, true);
-  Object.entries(extensionPacks).forEach(([name, paths]) => {
-    // We are guaranteed that there is at least one path found for each extension pack.
-    // If there are multiple paths, then we have a problem. This means that there is
-    // ambiguity in which path to use. This is an error.
-    if (paths.length > 1) {
-      throw new Error(
-        `Multiple versions of extension pack '${name}' found: ${paths.join(
-          ", ",
-        )}`,
-      );
-    }
+  extensionPacks.forEach((name) => {
     // Add this extension pack as a dependency. It doesn't matter which
     // version we specify, since we are guaranteed that the extension pack
     // is resolved from source at the given path.
@@ -437,7 +507,6 @@ async function injectExtensionPacks(
   syntheticQueryPack.dependencies = dependencies;
 
   await writeFile(qlpackFile, dump(syntheticQueryPack));
-  await cliServer.clearCache();
 }
 
 function updateDefaultSuite(qlpack: QlPackFile, packRelativePath: string) {

extensions/ql-vscode/test/matchers/toExistInCodeQLPack.ts

@@ -0,0 +1,50 @@
+import { expect } from "@jest/globals";
+import type { MatcherFunction } from "expect";
+import type { QueryPackFS } from "../vscode-tests/utils/bundled-pack-helpers";
+import { EOL } from "os";
+
+/**
+ * Custom Jest matcher to check if a file exists in a query pack.
+ */
+// eslint-disable-next-line func-style -- We need to set the type of this function
+const toExistInCodeQLPack: MatcherFunction<[packFS: QueryPackFS]> = function (
+  actual,
+  packFS,
+) {
+  if (typeof actual !== "string") {
+    throw new TypeError(
+      `Expected actual value to be a string. Found ${typeof actual}`,
+    );
+  }
+
+  const pass = packFS.fileExists(actual);
+  if (pass) {
+    return {
+      pass: true,
+      message: () => `expected ${actual} not to exist in pack`,
+    };
+  } else {
+    const files = packFS.allFiles();
+    const filesString = files.length > 0 ? files.join(EOL) : "<none>";
+    return {
+      pass: false,
+      message: () =>
+        `expected ${actual} to exist in pack.\nThe following files were found in the pack:\n${filesString}`,
+    };
+  }
+};
+
+expect.extend({ toExistInCodeQLPack });
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace -- We need to extend this global declaration
+  namespace jest {
+    interface AsymmetricMatchers {
+      toExistInCodeQLPack(packFS: QueryPackFS): void;
+    }
+
+    interface Matchers<R> {
+      toExistInCodeQLPack(packFS: QueryPackFS): R;
+    }
+  }
+}

extensions/ql-vscode/test/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml

@@ -1,4 +1,4 @@
 name: github/remote-query-pack
 version: 0.0.0
 dependencies:
-  codeql/javascript-all: ${workspace}
+  codeql/javascript-all: "*"

extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts

@@ -24,6 +24,8 @@ import { readBundledPack } from "../../utils/bundled-pack-helpers";
 import { load } from "js-yaml";
 import type { ExtensionPackMetadata } from "../../../../src/model-editor/extension-pack-metadata";
 import type { QlPackLockFile } from "../../../../src/packaging/qlpack-lock-file";
+//import { expect } from "@jest/globals";
+import "../../../matchers/toExistInCodeQLPack";
 
 describe("Variant Analysis Manager", () => {
   let cli: CodeQLCliServer;
@@ -331,14 +333,14 @@ describe("Variant Analysis Manager", () => {
 
       const packFS = await readBundledPack(request.query.pack);
       filesThatExist.forEach((file) => {
-        expect(packFS.fileExists(file)).toBe(true);
+        expect(file).toExistInCodeQLPack(packFS);
       });
 
       qlxFilesThatExist.forEach((file) => {
-        expect(packFS.fileExists(file)).toBe(true);
+        expect(file).toExistInCodeQLPack(packFS);
       });
       filesThatDoNotExist.forEach((file) => {
-        expect(packFS.fileExists(file)).toBe(false);
+        expect(file).not.toExistInCodeQLPack(packFS);
       });
 
       expect(
@@ -364,9 +366,17 @@ describe("Variant Analysis Manager", () => {
 
       // Assume the first dependency to check is the core library.
       if (dependenciesToCheck.length > 0) {
-        expect(qlpackContents.dependencies?.[dependenciesToCheck[0]]).toEqual(
-          "*",
-        );
+        const dependencyVersion =
+          qlpackContents.dependencies?.[dependenciesToCheck[0]];
+
+        // There should be a version specified.
+        expect(dependencyVersion).toBeDefined();
+
+        // Any `${workspace}` placeholder should have been replaced.
+        // The actual version might be `*` (for the legacy code path where we replace workspace
+        // references with `*`) or a specific version (for the new code path where the CLI does all
+        // the work).
+        expect(dependencyVersion).not.toEqual("${workspace}");
       }
       const qlpackLockContents = load(
         packFS.fileContents("codeql-pack.lock.yml").toString("utf-8"),

extensions/ql-vscode/test/vscode-tests/cli.ts

@@ -1,3 +1,4 @@
+import { basename } from "path";
 import { workspace } from "vscode";
 
 /**
@@ -6,7 +7,10 @@ import { workspace } from "vscode";
  */
 function hasCodeQL() {
   const folders = workspace.workspaceFolders;
-  return !!folders?.some((folder) => folder.uri.path.endsWith("/codeql"));
+  return !!folders?.some((folder) => {
+    const name = basename(folder.uri.fsPath);
+    return name === "codeql" || name === "ql";
+  });
 }
 
 // describeWithCodeQL will be equal to describe if the CodeQL libraries are

extensions/ql-vscode/test/vscode-tests/utils/bundled-pack-helpers.ts

@@ -4,10 +4,11 @@ import { extract as tar_extract } from "tar-stream";
 import { pipeline } from "stream/promises";
 import { createGunzip } from "zlib";
 
-interface QueryPackFS {
+export interface QueryPackFS {
   fileExists: (name: string) => boolean;
   fileContents: (name: string) => Buffer;
   directoryContents: (name: string) => string[];
+  allFiles: () => string[];
 }
 
 export async function readBundledPack(
@@ -82,5 +83,8 @@ export async function readBundledPack(
         )
         .map((dir) => dir.substring(name.length + 1));
     },
+    allFiles: (): string[] => {
+      return Object.keys(files);
+    },
   };
 }