Merge pull request #985 from github/qc-packs

Remote Queries: Create packs for remote queries

extensions/ql-vscode/src/cli.ts

@@ -806,6 +806,39 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  async packInstall(dir: string) {
+    return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
+  }
+
+  async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
+    const args = [
+      '-o',
+      outputPath,
+      dir,
+      '--additional-packs',
+      workspaceFolders.join(path.delimiter)
+    ];
+    if (!precompile && await this.cliConstraints.supportsNoPrecompile()) {
+      args.push('--no-precompile');
+    }
+
+    return this.runJsonCodeQlCliCommand(['pack', 'bundle'], args, 'Bundling pack');
+  }
+
+  async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
+    const args = includeQueries ? [dir] : ['--no-include-queries', dir];
+    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
+    // previous versions return a list of packs.
+    const results: { paths: string[] } | string[] = await this.runJsonCodeQlCliCommand(['pack', 'packlist'], args, 'Generating the pack list');
+
+    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
+    if ('paths' in results) {
+      return results.paths;
+    } else {
+      return results;
+    }
+  }
+
   async generateDil(qloFile: string, outFile: string): Promise<void> {
     const extraArgs = await this.cliConstraints.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
@@ -1099,6 +1132,16 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_DATABASE_UNBUNDLE = new SemVer('2.6.0');
 
+  /**
+   * CLI version where the `--no-precompile` option for pack creation was introduced.
+   */
+  public static CLI_VERSION_WITH_NO_PRECOMPILE = new SemVer('2.7.1');
+
+  /**
+   * CLI version where remote queries are supported.
+   */
+  public static CLI_VERSION_REMOTE_QUERIES = new SemVer('2.6.3');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1135,4 +1178,12 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
   }
 
+  async supportsNoPrecompile() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NO_PRECOMPILE);
+  }
+
+  async supportsRemoteQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES);
+  }
+
 }

extensions/ql-vscode/src/config.ts

@@ -304,7 +304,7 @@ const REMOTE_QUERIES_SETTING = new Setting('remoteQueries', ROOT_SETTING);
 /**
  * Lists of GitHub repositories that you want to query remotely via the "Run Remote query" command.
  * Note: This command is only available for internal users.
- * 
+ *
  * This setting should be a JSON object where each key is a user-specified name (string),
  * and the value is an array of GitHub repositories (of the form `<owner>/<repo>`).
  */
@@ -314,6 +314,10 @@ export function getRemoteRepositoryLists(): Record<string, string[]> | undefined
   return REMOTE_REPO_LISTS.getValue<Record<string, string[]>>() || undefined;
 }
 
+export async function setRemoteRepositoryLists(lists: Record<string, string[]> | undefined) {
+  await REMOTE_REPO_LISTS.updateValue(lists, ConfigurationTarget.Global);
+}
+
 /**
  * The name of the "controller" repository that you want to use with the "Run Remote query" command.
  * Note: This command is only available for internal users.

extensions/ql-vscode/src/extension.ts

@@ -718,13 +718,25 @@ async function activateWithInstalledDistribution(
   );
   // The "runRemoteQuery" command is internal-only.
   ctx.subscriptions.push(
-    commandRunner('codeQL.runRemoteQuery', async (
+    commandRunnerWithProgress('codeQL.runRemoteQuery', async (
+      progress: ProgressCallback,
+      token: CancellationToken,
       uri: Uri | undefined
     ) => {
       if (isCanary()) {
+        progress({
+          maxStep: 5,
+          step: 0,
+          message: 'Getting credentials'
+        });
         const credentials = await Credentials.initialize(ctx);
-        await runRemoteQuery(cliServer, credentials, uri || window.activeTextEditor?.document.uri);
+        await runRemoteQuery(cliServer, credentials, uri || window.activeTextEditor?.document.uri, false, progress, token);
+      } else {
+        throw new Error('Remote queries require the CodeQL Canary version to run.');
       }
+    }, {
+      title: 'Run Remote Query',
+      cancellable: true
     })
   );
   ctx.subscriptions.push(

extensions/ql-vscode/src/helpers.ts

@@ -10,6 +10,7 @@ import {
   env
 } from 'vscode';
 import { CodeQLCliServer, QlpacksInfo } from './cli';
+import { UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 
 /**
@@ -494,14 +495,25 @@ export async function findLanguage(
       void logger.log('Could not autodetect query language. Select language manually.');
     }
   }
-  const availableLanguages = Object.keys(await cliServer.resolveLanguages());
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+
+export async function askForLanguage(cliServer: CodeQLCliServer, throwOnEmpty = true): Promise<string | undefined> {
+  const availableLanguages = Object.keys(await cliServer.resolveLanguages()).sort();
   const language = await Window.showQuickPick(
     availableLanguages,
     { placeHolder: 'Select target language for your query', ignoreFocusOut: true }
   );
   if (!language) {
     // This only happens if the user cancels the quick pick.
-    void showAndLogErrorMessage('Language not found. Language must be specified manually.');
+    if (throwOnEmpty) {
+      throw new UserCancellationException('Cancelled.');
+    } else {
+      void showAndLogErrorMessage('Language not found. Language must be specified manually.');
+    }
   }
   return language;
 }

extensions/ql-vscode/src/quick-query.ts

@@ -118,7 +118,7 @@ export async function displayQuickQuery(
     // Only rewrite the qlpack file if the database has changed
     if (shouldRewrite) {
       const quickQueryQlpackYaml: any = {
-        name: 'quick-query',
+        name: 'vscode/quick-query',
         version: '1.0.0',
         libraryPathDependencies: [qlpack]
       };

extensions/ql-vscode/src/run-remote-query.ts

@@ -1,11 +1,15 @@
-import { QuickPickItem, Uri, window } from 'vscode';
+import { CancellationToken, QuickPickItem, Uri, window } from 'vscode';
+import * as path from 'path';
 import * as yaml from 'js-yaml';
 import * as fs from 'fs-extra';
-import { findLanguage, showAndLogErrorMessage, showAndLogInformationMessage, showInformationMessageWithAction } from './helpers';
+import * as tmp from 'tmp-promise';
+import { askForLanguage, findLanguage, getOnDiskWorkspaceFolders, showAndLogErrorMessage, showAndLogInformationMessage, showInformationMessageWithAction } from './helpers';
 import { Credentials } from './authentication';
 import * as cli from './cli';
 import { logger } from './logging';
 import { getRemoteControllerRepo, getRemoteRepositoryLists, setRemoteControllerRepo } from './config';
+import { tmpDir } from './run-queries';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
 interface Config {
   repositories: string[];
   ref?: string;
@@ -68,97 +72,313 @@ export async function getRepositories(): Promise<string[] | undefined> {
   }
 }
 
-export async function runRemoteQuery(cliServer: cli.CodeQLCliServer, credentials: Credentials, uri?: Uri) {
-  if (!uri?.fsPath.endsWith('.ql')) {
+/**
+ * Two possibilities:
+ * 1. There is no qlpack.yml in this directory. Assume this is a lone query and generate a synthetic qlpack for it.
+ * 2. There is a qlpack.yml in this directory. Assume this is a query pack and use the yml to pack the query before uploading it.
+ *
+ * @returns the entire qlpack as a base64 string.
+ */
+async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: string, queryPackDir: string, fallbackLanguage?: string): Promise<{
+  base64Pack: string,
+  language: string
+}> {
+  const originalPackRoot = path.dirname(queryFile);
+  // TODO this assumes that the qlpack.yml is in the same directory as the query file, but in reality,
+  // the file could be in a parent directory.
+  const targetQueryFileName = path.join(queryPackDir, path.basename(queryFile));
+
+  // the server is expecting the query file to be named `query.ql`. Rename it here.
+  const renamedQueryFile = path.join(queryPackDir, 'query.ql');
+
+  let language: string | undefined;
+  if (await fs.pathExists(path.join(originalPackRoot, 'qlpack.yml'))) {
+    // don't include ql files. We only want the queryFile to be copied.
+    const toCopy = await cliServer.packPacklist(originalPackRoot, false);
+
+    // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist.
+    [path.join(originalPackRoot, 'qlpack.lock.yml'), path.join(originalPackRoot, 'codeql-pack.lock.yml'), queryFile]
+      .forEach(absolutePath => {
+        if (absolutePath) {
+          toCopy.push(absolutePath);
+        }
+      });
+
+    let copiedCount = 0;
+    await fs.copy(originalPackRoot, queryPackDir, {
+      filter: (file: string) =>
+        // copy file if it is in the packlist, or it is a parent directory of a file in the packlist
+        !!toCopy.find(f => {
+          // Normalized paths ensure that Windows drive letters are capitalized consistently.
+          const normalizedPath = Uri.file(f).fsPath;
+          const matches = normalizedPath === file || normalizedPath.startsWith(file + path.sep);
+          if (matches) {
+            copiedCount++;
+          }
+          return matches;
+        })
+    });
+
+    // ensure the qlpack.yml has a valid name
+    await ensureQueryPackName(queryPackDir);
+
+    void logger.log(`Copied ${copiedCount} files to ${queryPackDir}`);
+
+    language = await findLanguage(cliServer, Uri.file(targetQueryFileName));
+
+  } else {
+    // open popup to ask for language if not already hardcoded
+    language = fallbackLanguage || await askForLanguage(cliServer);
+
+    // copy only the query file to the query pack directory
+    // and generate a synthetic query pack
+    // TODO this has a limitation that query packs inside of a workspace will not resolve its peer dependencies.
+    // Something to work on later. For now, we will only support query packs that are not in a workspace.
+    void logger.log(`Copying ${queryFile} to ${queryPackDir}`);
+    await fs.copy(queryFile, targetQueryFileName);
+    void logger.log('Generating synthetic query pack');
+    const syntheticQueryPack = {
+      name: 'codeql-remote/query',
+      version: '0.0.0',
+      dependencies: {
+        [`codeql/${language}-all`]: '*',
+      }
+    };
+    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(syntheticQueryPack));
+  }
+  if (!language) {
+    throw new UserCancellationException('Could not determine language.');
+  }
+
+  await fs.rename(targetQueryFileName, renamedQueryFile);
+
+  const bundlePath = await getPackedBundlePath(queryPackDir);
+  void logger.log(`Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`);
+  await cliServer.packInstall(queryPackDir);
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+  await cliServer.packBundle(queryPackDir, workspaceFolders, bundlePath, false);
+  const base64Pack = (await fs.readFile(bundlePath)).toString('base64');
+  return {
+    base64Pack,
+    language
+  };
+}
+
+/**
+ * Ensure that the qlpack.yml has a valid name. For local purposes,
+ * Anonymous packs and names that are not prefixed by a scope (ie `<foo>/`)
+ * are sufficient. But in order to create a pack, the name must be prefixed.
+ *
+ * @param queryPackDir the directory containing the query pack.
+ */
+async function ensureQueryPackName(queryPackDir: string) {
+  const pack = yaml.safeLoad(await fs.readFile(path.join(queryPackDir, 'qlpack.yml'), 'utf8')) as { name: string; };
+  if (!pack.name || !pack.name.includes('/')) {
+    if (!pack.name) {
+      pack.name = 'codeql-remote/query';
+    } else if (!pack.name.includes('/')) {
+      pack.name = `codeql-remote/${pack.name}`;
+    }
+    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(pack));
+  }
+}
+
+async function createRemoteQueriesTempDirectory() {
+  const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
+  const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
+  await fs.mkdirp(queryPackDir);
+  return { remoteQueryDir, queryPackDir };
+}
+
+async function getPackedBundlePath(queryPackDir: string) {
+  return tmp.tmpName({
+    dir: path.dirname(queryPackDir),
+    postfix: 'generated.tgz',
+    prefix: 'qlpack',
+  });
+}
+
+export async function runRemoteQuery(
+  cliServer: cli.CodeQLCliServer,
+  credentials: Credentials,
+  uri: Uri | undefined,
+  dryRun: boolean,
+  progress: ProgressCallback,
+  token: CancellationToken
+): Promise<void | string> {
+  if (!(await cliServer.cliConstraints.supportsRemoteQueries())) {
+    throw new Error(`Remote queries are not supported by this version of CodeQL. Please upgrade to v${cli.CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
+      } or later.`);
+  }
+
+  const { remoteQueryDir, queryPackDir } = await createRemoteQueriesTempDirectory();
+  try {
+    if (!uri?.fsPath.endsWith('.ql')) {
+      throw new UserCancellationException('Not a CodeQL query file.');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 1,
+      message: 'Determining project list'
+    });
+
+    const queryFile = uri.fsPath;
+    const repositoriesFile = queryFile.substring(0, queryFile.length - '.ql'.length) + '.repositories';
+    let ref: string | undefined;
+    // For the case of single file remote queries, use the language from the config in order to avoid the user having to select it.
+    let fallbackLanguage: string | undefined;
+    let repositories: string[] | undefined;
+
+    progress({
+      maxStep: 5,
+      step: 2,
+      message: 'Determining query target language'
+    });
+
+    // If the user has an explicit `.repositories` file, use that.
+    // Otherwise, prompt user to select repositories from the `codeQL.remoteQueries.repositoryLists` setting.
+    if (await fs.pathExists(repositoriesFile)) {
+      void logger.log(`Found '${repositoriesFile}'. Using information from that file to run ${queryFile}.`);
+
+      const config = yaml.safeLoad(await fs.readFile(repositoriesFile, 'utf8')) as Config;
+
+      ref = config.ref || 'main';
+      fallbackLanguage = config.language;
+      repositories = config.repositories;
+    } else {
+      ref = 'main';
+      repositories = await getRepositories();
+    }
+
+    if (!repositories || repositories.length === 0) {
+      throw new UserCancellationException('No repositories to query.');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 3,
+      message: 'Determining controller repo'
+    });
+
+    // Get the controller repo from the config, if it exists.
+    // If it doesn't exist, prompt the user to enter it, and save that value to the config.
+    let controllerRepo: string | undefined;
+    controllerRepo = getRemoteControllerRepo();
+    if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
+      void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
+      controllerRepo = await window.showInputBox({
+        title: 'Controller repository in which to display progress and results of remote queries',
+        placeHolder: '<owner>/<repo>',
+        prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
+        ignoreFocusOut: true,
+      });
+      if (!controllerRepo) {
+        void showAndLogErrorMessage('No controller repository entered.');
+        return;
+      } else if (!REPO_REGEX.test(controllerRepo)) { // Check if user entered invalid input
+        void showAndLogErrorMessage('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
+        return;
+      }
+      void logger.log(`Setting the controller repository as: ${controllerRepo}`);
+      await setRemoteControllerRepo(controllerRepo);
+    }
+
+    void logger.log(`Using controller repository: ${controllerRepo}`);
+    const [owner, repo] = controllerRepo.split('/');
+
+    progress({
+      maxStep: 5,
+      step: 4,
+      message: 'Bundling the query pack'
+    });
+
+    if (token.isCancellationRequested) {
+      throw new UserCancellationException('Cancelled');
+    }
+
+    const { base64Pack, language } = await generateQueryPack(cliServer, queryFile, queryPackDir, fallbackLanguage);
+
+    if (token.isCancellationRequested) {
+      throw new UserCancellationException('Cancelled');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 5,
+      message: 'Sending request'
+    });
+
+    await runRemoteQueriesApiRequest(credentials, ref, language, repositories, owner, repo, base64Pack, dryRun);
+
+    if (dryRun) {
+      return remoteQueryDir.path;
+    } else {
+      // don't return the path because it has been deleted
+      return;
+    }
+
+  } finally {
+    if (dryRun) {
+      // If we are in a dry run keep the data around for debugging purposes.
+      void logger.log(`[DRY RUN] Not deleting ${queryPackDir}.`);
+    } else {
+      await remoteQueryDir.cleanup();
+    }
+  }
+}
+
+async function runRemoteQueriesApiRequest(
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repositories: string[],
+  owner: string,
+  repo: string,
+  queryPackBase64: string,
+  dryRun = false
+): Promise<void> {
+
+  if (dryRun) {
+    void showAndLogInformationMessage('[DRY RUN] Would have sent request. See extension log for the payload.');
+    void logger.log(JSON.stringify({ ref, language, repositories, owner, repo, queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes' }));
     return;
   }
 
-  const queryFile = uri.fsPath;
-  const query = await fs.readFile(queryFile, 'utf8');
-
-  const repositoriesFile = queryFile.substring(0, queryFile.length - '.ql'.length) + '.repositories';
-  let ref: string | undefined;
-  let language: string | undefined;
-  let repositories: string[] | undefined;
-
-  // If the user has an explicit `.repositories` file, use that.
-  // Otherwise, prompt user to select repositories from the `codeQL.remoteQueries.repositoryLists` setting.
-  if (await fs.pathExists(repositoriesFile)) {
-    void logger.log(`Found '${repositoriesFile}'. Using information from that file to run ${queryFile}.`);
-
-    const config = yaml.safeLoad(await fs.readFile(repositoriesFile, 'utf8')) as Config;
-
-    ref = config.ref || 'main';
-    language = config.language || await findLanguage(cliServer, uri);
-    repositories = config.repositories;
-  } else {
-    ref = 'main';
-    [language, repositories] = await Promise.all([findLanguage(cliServer, uri), getRepositories()]);
-  }
-
-  if (!language) {
-    return; // No error message needed, since `findLanguage` already displays one.
-  }
-
-  if (!repositories || repositories.length === 0) {
-    return; // No error message needed, since `getRepositories` already displays one.
-  }
-
-  // Get the controller repo from the config, if it exists.
-  // If it doesn't exist, prompt the user to enter it, and save that value to the config.
-  let controllerRepo: string | undefined;
-  controllerRepo = getRemoteControllerRepo();
-  if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
-    void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
-    controllerRepo = await window.showInputBox({
-      title: 'Controller repository in which to display progress and results of remote queries',
-      placeHolder: '<owner>/<repo>',
-      prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
-      ignoreFocusOut: true,
-    });
-    if (!controllerRepo) {
-      void showAndLogErrorMessage('No controller repository entered.');
-      return;
-    } else if (!REPO_REGEX.test(controllerRepo)) { // Check if user entered invalid input
-      void showAndLogErrorMessage('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
-      return;
-    }
-    void logger.log(`Setting the controller repository as: ${controllerRepo}`);
-    await setRemoteControllerRepo(controllerRepo);
-  }
-
-  void logger.log(`Using controller repository: ${controllerRepo}`);
-  const [owner, repo] = controllerRepo.split('/');
-
-  await runRemoteQueriesApiRequest(credentials, ref, language, repositories, query, owner, repo);
-}
-
-async function runRemoteQueriesApiRequest(credentials: Credentials, ref: string, language: string, repositories: string[], query: string, owner: string, repo: string) {
-  const octokit = await credentials.getOctokit();
-
   try {
+    const octokit = await credentials.getOctokit();
     await octokit.request(
       'POST /repos/:owner/:repo/code-scanning/codeql/queries',
       {
         owner,
         repo,
         data: {
-          ref: ref,
-          language: language,
-          repositories: repositories,
-          query: query,
+          ref,
+          language,
+          repositories,
+          query_pack: queryPackBase64,
         }
       }
     );
     void showAndLogInformationMessage(`Successfully scheduled runs. [Click here to see the progress](https://github.com/${owner}/${repo}/actions).`);
 
   } catch (error) {
-    await attemptRerun(error, credentials, ref, language, repositories, query, owner, repo);
+    await attemptRerun(error, credentials, ref, language, repositories, owner, repo, queryPackBase64, dryRun);
   }
 }
 
 /** Attempts to rerun the query on only the valid repositories */
-export async function attemptRerun(error: any, credentials: Credentials, ref: string, language: string, repositories: string[], query: string, owner: string, repo: string) {
+export async function attemptRerun(
+  error: any,
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repositories: string[],
+  owner: string,
+  repo: string,
+  queryPackBase64: string,
+  dryRun = false
+) {
   if (typeof error.message === 'string' && error.message.includes('Some repositories were invalid')) {
     const invalidRepos = error?.response?.data?.invalid_repos || [];
     const reposWithoutDbUploads = error?.response?.data?.repos_without_db_uploads || [];
@@ -181,11 +401,9 @@ export async function attemptRerun(error: any, credentials: Credentials, ref: st
     if (rerunQuery) {
       const validRepositories = repositories.filter(r => !invalidRepos.includes(r) && !reposWithoutDbUploads.includes(r));
       void logger.log(`Rerunning query on set of valid repositories: ${JSON.stringify(validRepositories)}`);
-      await runRemoteQueriesApiRequest(credentials, ref, language, validRepositories, query, owner, repo);
+      await runRemoteQueriesApiRequest(credentials, ref, language, validRepositories, owner, repo, queryPackBase64, dryRun);
     }
-
   } else {
     void showAndLogErrorMessage(error);
   }
-
 }

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-no-qlpack/in-pack.ql

@@ -0,0 +1,2 @@
+// This file should not be included the remote query pack.
+select 1

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-no-qlpack/lib.qll

@@ -0,0 +1,3 @@
+int number() {
+  result = 1
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-no-qlpack/not-in-pack.ql

@@ -0,0 +1,4 @@
+import javascript
+import lib
+
+select number()

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/in-pack.ql

@@ -0,0 +1,4 @@
+import javascript
+import lib
+
+select number()

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/lib.qll

@@ -0,0 +1,3 @@
+int number() {
+  result = 1
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/not-in-pack.ql

@@ -0,0 +1,2 @@
+// This file should not be included the remote query pack.
+select 1

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml

@@ -0,0 +1,5 @@
+name: github/remote-query-pack
+version: 0.0.0
+extractor: javascript
+dependencies:
+  codeql/javascript-all: '*'

extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts

@@ -1,6 +1,10 @@
-import 'mocha';
-import 'sinon-chai';
 import { runTestsInDirectory } from '../index-template';
+import 'mocha';
+import * as sinonChai from 'sinon-chai';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+chai.use(chaiAsPromised);
+chai.use(sinonChai);
 
 // The simple database used throughout the tests
 export function run(): Promise<void> {

extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts

@@ -0,0 +1,194 @@
+import { assert, expect } from 'chai';
+import * as path from 'path';
+import * as sinon from 'sinon';
+import { CancellationToken, extensions, QuickPickItem, Uri, window } from 'vscode';
+import 'mocha';
+import * as fs from 'fs-extra';
+import * as yaml from 'js-yaml';
+
+import { runRemoteQuery } from '../../run-remote-query';
+import { Credentials } from '../../authentication';
+import { CliVersionConstraint, CodeQLCliServer } from '../../cli';
+import { CodeQLExtensionInterface } from '../../extension';
+import { setRemoteControllerRepo, setRemoteRepositoryLists } from '../../config';
+import { UserCancellationException } from '../../commandRunner';
+
+describe('Remote queries', function() {
+  const baseDir = path.join(__dirname, '../../../src/vscode-tests/cli-integration');
+
+  let sandbox: sinon.SinonSandbox;
+
+  // up to 3 minutes per test
+  this.timeout(3 * 60 * 1000);
+
+  let cli: CodeQLCliServer;
+  let credentials: Credentials = {} as unknown as Credentials;
+  let token: CancellationToken;
+  let progress: sinon.SinonSpy;
+  let showQuickPickSpy: sinon.SinonStub;
+
+  // use `function` so we have access to `this`
+  beforeEach(async function() {
+    sandbox = sinon.createSandbox();
+
+    const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+    if ('cliServer' in extension) {
+      cli = extension.cliServer;
+    } else {
+      throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+    }
+
+    if (!(await cli.cliConstraints.supportsRemoteQueries())) {
+      console.log(`Remote queries are not supported on CodeQL CLI v${CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
+        }. Skipping this test.`);
+      this.skip();
+    }
+    credentials = {} as unknown as Credentials;
+    token = {
+      isCancellationRequested: false
+    } as unknown as CancellationToken;
+
+    progress = sandbox.spy();
+    // Should not have asked for a language
+    showQuickPickSpy = sandbox.stub(window, 'showQuickPick')
+      .onFirstCall().resolves({ repoList: ['github/vscode-codeql'] } as unknown as QuickPickItem)
+      .onSecondCall().resolves('javascript' as unknown as QuickPickItem);
+
+    // always run in the vscode-codeql repo
+    await setRemoteControllerRepo('github/vscode-codeql');
+    await setRemoteRepositoryLists({ 'vscode-codeql': ['github/vscode-codeql'] });
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it('should run a remote query that is part of a qlpack', async () => {
+    const fileUri = getFile('data-remote-qlpack/in-pack.ql');
+
+    const queryPackRootDir = (await runRemoteQuery(cli, credentials, fileUri, true, progress, token))!;
+    printDirectoryContents(queryPackRootDir);
+
+    // to retrieve the list of repositories
+    expect(showQuickPickSpy).to.have.been.calledOnce;
+
+    // check a few files that we know should exist and others that we know should not
+
+    // the tarball to deliver to the server
+    expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
+
+    const queryPackDir = path.join(queryPackRootDir, 'query-pack');
+    printDirectoryContents(queryPackDir);
+
+    // in-pack.ql renamed to query.ql
+    expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
+
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // the compiled pack
+    const compiledPackDir = path.join(queryPackDir, '.codeql/pack/github/remote-query-pack/0.0.0/');
+    printDirectoryContents(compiledPackDir);
+
+    expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // dependencies
+    const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+    const packNames = fs.readdirSync(libraryDir).sort();
+    expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']);
+  });
+
+  it('should run a remote query that is not part of a qlpack', async () => {
+    const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
+
+    const queryPackRootDir = (await runRemoteQuery(cli, credentials, fileUri, true, progress, token))!;
+
+    // to retrieve the list of repositories
+    // and a second time to ask for the language
+    expect(showQuickPickSpy).to.have.been.calledTwice;
+
+    // check a few files that we know should exist and others that we know should not
+
+    // the tarball to deliver to the server
+    printDirectoryContents(queryPackRootDir);
+    expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
+
+    const queryPackDir = path.join(queryPackRootDir, 'query-pack');
+    printDirectoryContents(queryPackDir);
+    // in-pack.ql renamed to query.ql
+    expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
+    expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // the compiled pack
+    const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
+    printDirectoryContents(compiledPackDir);
+    expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
+    expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
+    // should have generated a correct qlpack file
+    const qlpackContents: any = yaml.safeLoad(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
+    expect(qlpackContents.name).to.equal('codeql-remote/query');
+    expect(qlpackContents.version).to.equal('0.0.0');
+    expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
+
+    // dependencies
+    const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+    printDirectoryContents(libraryDir);
+    const packNames = fs.readdirSync(libraryDir).sort();
+    expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']);
+  });
+
+  it('should cancel a run before uploading', async () => {
+    const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
+
+    const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+
+    token.isCancellationRequested = true;
+
+    try {
+      await promise;
+      assert.fail('should have thrown');
+    } catch (e) {
+      expect(e).to.be.instanceof(UserCancellationException);
+    }
+  });
+
+  function getFile(file: string): Uri {
+    return Uri.file(path.join(baseDir, file));
+  }
+
+  function printDirectoryContents(dir: string) {
+    console.log(`DIR ${dir}`);
+    if (!fs.existsSync(dir)) {
+      console.log(`DIR ${dir} does not exist`);
+    }
+    fs.readdirSync(dir).sort().forEach(f => console.log(`  ${f}`));
+  }
+});

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -58,6 +58,12 @@ export async function ensureCli(useCli: boolean) {
       return;
     }
 
+    if ('CODEQL_PATH' in process.env) {
+      const executablePath = process.env.CODEQL_PATH;
+      console.log(`Using existing CLI at ${executablePath}`);
+      return;
+    }
+
     const assetName = DistributionManager.getRequiredAssetName();
     const url = getCliDownloadUrl(assetName);
     const unzipDir = getCliUnzipDir();