v1.3.10

Also, remove an errant `only`, which was preventing some tests from
running.

.github/workflows/release.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd extensions/ql-vscode
-          npm install
+          npm ci
         shell: bash
 
       - name: Build
@@ -55,9 +55,6 @@ jobs:
           REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
-      # Uploading artifacts is not necessary to create a release.
-      # This is just in case the release itself fails and we want to access the built artifacts from Actions.
-      # TODO Remove if not useful.
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -126,3 +123,40 @@ jobs:
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
           base: main
+
+  vscode-publish:
+    name: Publish to VS Code Marketplace
+    needs: build
+    environment: publish-vscode-marketplace
+    runs-on: ubuntu-latest
+    env:
+      VSCE_TOKEN: ${{ secrets.VSCE_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix || \
+            echo "Failed to publish to VS Code Marketplace. \
+                  If this was an authentication problem, please make sure the \
+                  auth token hasn't expired."
+
+  open-vsx-publish:
+    name: Publish to Open VSX Registry
+    needs: build
+    environment: publish-open-vsx
+    runs-on: ubuntu-latest
+    env:
+      OPEN_VSX_TOKEN: ${{ secrets.OPEN_VSX_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx ovsx publish -p $OPEN_VSX_TOKEN *.vsix

.vscode/launch.json

@@ -8,7 +8,9 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        // Add a reference to a workspace to open. Eg-
+        // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -87,6 +89,9 @@
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
         "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
+        // Add a path to a checked out instance of the codeql repository so the libraries are
+        // available in the workspace for the tests.
+        // "${workspaceRoot}/../codeql"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,

CONTRIBUTING.md

@@ -114,14 +114,32 @@ Alternatively, you can run the tests inside of vscode. There are several vscode
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
-1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
-1. Click the `...` menu in the CodeQL row and click **Update**.
-1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+    - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
+1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
+1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
 1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
 1. If documentation changes need to be published, notify documentation team that release has been made.
 1. Review and merge the version bump PR that is automatically created by Actions.
 
+## Secrets and authentication for publishing
+
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+
+To regenerate the Open VSX token:
+
+1. Log in to the [user settings page on Open VSX](https://open-vsx.org/user-settings/namespaces).
+1. Make sure you are a member of the GitHub namespace.
+1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
+1. Update the secret in the `publish-open-vsx` environment in the project settings.
+
+To regenerate the VSCode Marketplace token:
+
+1. Follow the instructions on [getting a PAT for Azure DevOps](https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token).
+1. Update the secret in the `publish-vscode-marketplace` environment in the project settings.
+
+Not that Azure DevOps PATs expire yearly and must be regenerated.
+
 ## Resources
 
 * [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,13 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.3.10 - 20 January 2021
+
+- Include the full stack in error log messages to help with debugging. [#726](https://github.com/github/vscode-codeql/pull/726)
+
+## 1.3.9 - 12 January 2021
+
+- No changes visible to end users.
+
 ## 1.3.8 - 17 December 2020
 
 - Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting the IDE. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)

extensions/ql-vscode/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vscode-codeql",
-  "version": "1.3.8",
+  "version": "1.3.9",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -217,9 +217,9 @@
       "dev": true
     },
     "@types/fs-extra": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.3.tgz",
-      "integrity": "sha512-NKdGoXLTFTRED3ENcfCsH8+ekV4gbsysanx2OPbstXVV6fZMgUCqTxubs6I9r7pbOJbFgVq1rpFtLURjKCZWUw==",
+      "version": "9.0.6",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.6.tgz",
+      "integrity": "sha512-ecNRHw4clCkowNOBJH1e77nvbPxHYnWIXMv1IAoG/9+MYGkgoyr3Ppxr7XYFNL41V422EDhyV4/4SSK8L2mlig==",
       "dev": true,
       "requires": {
         "@types/node": "*"

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.3.8",
+  "version": "1.3.10",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -695,7 +695,7 @@
     "viewsWelcome": [
       {
         "view": "codeQLAstViewer",
-        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a CodeQL database.\n[View AST](command:codeQL.viewAst)"
       },
       {
         "view": "codeQLQueryHistory",
@@ -703,7 +703,7 @@
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a Code QL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
       }
     ]
   },
@@ -746,7 +746,7 @@
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^9.0.3",
+    "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",

extensions/ql-vscode/src/astViewer.ts

@@ -18,7 +18,7 @@ import { DatabaseItem } from './databases';
 import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
 import { showLocation } from './interface-utils';
 import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
-import { commandRunner } from './helpers';
+import { commandRunner } from './commandRunner';
 import { DisposableObject } from './vscode-utils/disposable-object';
 
 export interface AstItem {

extensions/ql-vscode/src/cli.ts

@@ -267,7 +267,7 @@ export class CodeQLCliServer implements Disposable {
       const argsString = args.join(' ');
       this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
-        await new Promise((resolve, reject) => {
+        await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
           process.stdout.addListener('data', (newData: Buffer) => {
             stdoutBuffers.push(newData);

extensions/ql-vscode/src/commandRunner.ts

@@ -0,0 +1,227 @@
+import {
+  CancellationToken,
+  ProgressOptions,
+  window as Window,
+  commands,
+  Disposable,
+  ProgressLocation
+} from 'vscode';
+import { showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
+import { logger } from './logging';
+
+export class UserCancellationException extends Error {
+  /**
+   * @param message The error message
+   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
+   */
+  constructor(message?: string, public readonly silent = false) {
+    super(message);
+  }
+}
+
+export interface ProgressUpdate {
+  /**
+   * The current step
+   */
+  step: number;
+  /**
+   * The maximum step. This *should* be constant for a single job.
+   */
+  maxStep: number;
+  /**
+   * The current progress message
+   */
+  message: string;
+}
+
+export type ProgressCallback = (p: ProgressUpdate) => void;
+
+/**
+ * A task that handles command invocations from `commandRunner`
+ * and includes a progress monitor.
+ *
+ *
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `ProgressTask` instance.
+ *
+ * @param progress a progress handler function. Call this
+ * function with a `ProgressUpdate` instance in order to
+ * denote some progress being achieved on this task.
+ * @param token a cencellation token
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+export type ProgressTask<R> = (
+  progress: ProgressCallback,
+  token: CancellationToken,
+  ...args: any[]
+) => Thenable<R>;
+
+/**
+ * A task that handles command invocations from `commandRunner`.
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `NoProgressTask` instance.
+ *
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+type NoProgressTask = ((...args: any[]) => Promise<any>);
+
+/**
+ * This mediates between the kind of progress callbacks we want to
+ * write (where we *set* current progress position and give
+ * `maxSteps`) and the kind vscode progress api expects us to write
+ * (which increment progress by a certain amount out of 100%).
+ *
+ * Where possible, the `commandRunner` function below should be used
+ * instead of this function. The commandRunner is meant for wrapping
+ * top-level commands and provides error handling and other support
+ * automatically.
+ *
+ * Only use this function if you need a progress monitor and the
+ * control flow does not always come from a command (eg- during
+ * extension activation, or from an internal language server
+ * request).
+ */
+export function withProgress<R>(
+  options: ProgressOptions,
+  task: ProgressTask<R>,
+  ...args: any[]
+): Thenable<R> {
+  let progressAchieved = 0;
+  return Window.withProgress(options,
+    (progress, token) => {
+      return task(p => {
+        const { message, step, maxStep } = p;
+        const increment = 100 * (step - progressAchieved) / maxStep;
+        progressAchieved = step;
+        progress.report({ message, increment });
+      }, token, ...args);
+    });
+}
+
+/**
+ * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
+ *
+ * In this variant of the command runner, no progress monitor is used.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task.
+ */
+export function commandRunner(
+  commandId: string,
+  task: NoProgressTask,
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    try {
+      return await task(...args);
+    } catch (e) {
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          logger.log(errorMessage);
+        } else {
+          showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
+      }
+      return undefined;
+    }
+  });
+}
+
+/**
+ * A generic wrapper for command registration.  This wrapper adds uniform error handling,
+ * progress monitoring, and cancellation for commands.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task after the progress callback
+ * and cancellation token.
+ * @param progressOptions Progress options to be sent to the progress monitor.
+ */
+export function commandRunnerWithProgress<R>(
+  commandId: string,
+  task: ProgressTask<R>,
+  progressOptions: Partial<ProgressOptions>
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const progressOptionsWithDefaults = {
+      location: ProgressLocation.Notification,
+      ...progressOptions
+    };
+    try {
+      return await withProgress(progressOptionsWithDefaults, task, ...args);
+    } catch (e) {
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          logger.log(errorMessage);
+        } else {
+          showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
+      }
+      return undefined;
+    }
+  });
+}
+
+/**
+ * Displays a progress monitor that indicates how much progess has been made
+ * reading from a stream.
+ *
+ * @param readable The stream to read progress from
+ * @param messagePrefix A prefix for displaying the message
+ * @param totalNumBytes Total number of bytes in this stream
+ * @param progress The progress callback used to set messages
+ */
+export function reportStreamProgress(
+  readable: NodeJS.ReadableStream,
+  messagePrefix: string,
+  totalNumBytes?: number,
+  progress?: ProgressCallback
+) {
+  if (progress && totalNumBytes) {
+    let numBytesDownloaded = 0;
+    const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+    const updateProgress = () => {
+      progress({
+        step: numBytesDownloaded,
+        maxStep: totalNumBytes,
+        message: `${messagePrefix} [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
+      });
+    };
+
+    // Display the progress straight away rather than waiting for the first chunk.
+    updateProgress();
+
+    readable.on('data', data => {
+      numBytesDownloaded += data.length;
+      updateProgress();
+    });
+  } else if (progress) {
+    progress({
+      step: 1,
+      maxStep: 2,
+      message: `${messagePrefix} (Size unknown)`,
+    });
+  }
+}

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -8,7 +8,7 @@ import fileRangeFromURI from './fileRangeFromURI';
 import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { QueryWithResults, compileAndRunQueryAgainstDatabase } from '../run-queries';
-import { ProgressCallback } from '../helpers';
+import { ProgressCallback } from '../commandRunner';
 import { KeyType } from './keyType';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
 

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -3,7 +3,8 @@ import * as vscode from 'vscode';
 import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
-import { CachedOperation, ProgressCallback, withProgress } from '../helpers';
+import { CachedOperation } from '../helpers';
+import { ProgressCallback, withProgress } from '../commandRunner';
 import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';

extensions/ql-vscode/src/databaseFetcher.ts

@@ -12,10 +12,12 @@ import * as path from 'path';
 
 import { DatabaseManager, DatabaseItem } from './databases';
 import {
-  reportStreamProgress,
-  ProgressCallback,
   showAndLogInformationMessage,
 } from './helpers';
+import {
+  reportStreamProgress,
+  ProgressCallback,
+} from './commandRunner';
 import { logger } from './logging';
 import { tmpDir } from './run-queries';
 

extensions/ql-vscode/src/databases-ui.ts

@@ -22,8 +22,10 @@ import {
 import {
   commandRunner,
   commandRunnerWithProgress,
-  getOnDiskWorkspaceFolders,
   ProgressCallback,
+} from './commandRunner';
+import {
+  getOnDiskWorkspaceFolders,
   showAndLogErrorMessage,
   isLikelyDatabaseRoot,
   isLikelyDbLanguageFolder

extensions/ql-vscode/src/databases.ts

@@ -9,9 +9,11 @@ import {
   showAndLogWarningMessage,
   showAndLogInformationMessage,
   isLikelyDatabaseRoot,
+} from './helpers';
+import {
   ProgressCallback,
   withProgress
-} from './helpers';
+} from './commandRunner';
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
 import { DisposableObject } from './vscode-utils/disposable-object';
 import { Logger, logger } from './logging';

extensions/ql-vscode/src/distribution.ts

@@ -7,10 +7,15 @@ import * as unzipper from 'unzipper';
 import * as url from 'url';
 import { ExtensionContext, Event } from 'vscode';
 import { DistributionConfig } from './config';
-import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from './helpers';
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage
+} from './helpers';
 import { logger } from './logging';
-import * as helpers from './helpers';
 import { getCodeQlCliVersion } from './cli-version';
+import { ProgressCallback, reportStreamProgress } from './commandRunner';
 
 /**
  * distribution.ts
@@ -221,7 +226,7 @@ export class DistributionManager implements DistributionProvider {
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     return this.extensionSpecificDistributionManager!.installDistributionRelease(release, progressCallback);
   }
 
@@ -303,14 +308,14 @@ class ExtensionSpecificDistributionManager {
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async installDistributionRelease(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     await this.downloadDistribution(release, progressCallback);
     // Store the installed release within the global extension state.
     this.storeInstalledRelease(release);
   }
 
   private async downloadDistribution(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     try {
       await this.removeDistribution();
     } catch (e) {
@@ -338,7 +343,7 @@ class ExtensionSpecificDistributionManager {
 
       const contentLength = assetStream.headers.get('content-length');
       const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
-      helpers.reportStreamProgress(assetStream.body, 'Downloading CodeQL CLI…', totalNumBytes, progressCallback);
+      reportStreamProgress(assetStream.body, 'Downloading CodeQL CLI…', totalNumBytes, progressCallback);
 
       await new Promise((resolve, reject) =>
         assetStream.body.pipe(archiveFile)
@@ -707,7 +712,9 @@ export async function getExecutableFromDirectory(directory: string, warnWhenNotF
 }
 
 function warnDeprecatedLauncher() {
-  helpers.showAndLogWarningMessage(
+
+  showAndLogWarningMessage(
+
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
     `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
   );

extensions/ql-vscode/src/extension.ts

@@ -45,6 +45,7 @@ import {
   GithubRateLimitedError
 } from './distribution';
 import * as helpers from './helpers';
+import { commandRunner, commandRunnerWithProgress, ProgressCallback, ProgressUpdate, withProgress } from './commandRunner';
 import { assertNever } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
 import { InterfaceManager } from './interface';
@@ -108,7 +109,7 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 
   stubbedCommands.forEach(command => {
     if (excludedCommands.indexOf(command) === -1) {
-      errorStubs.push(helpers.commandRunner(command, stubGenerator(command)));
+      errorStubs.push(commandRunner(command, stubGenerator(command)));
     }
   });
 }
@@ -194,7 +195,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
             location: ProgressLocation.Notification,
           };
 
-          await helpers.withProgress(progressOptions, progress =>
+          await withProgress(progressOptions, progress =>
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
@@ -308,7 +309,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     shouldDisplayMessageWhenNoUpdates: false,
     allowAutoUpdating: true
   })));
-  ctx.subscriptions.push(helpers.commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
     isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: true,
     allowAutoUpdating: true
@@ -430,7 +431,7 @@ async function activateWithInstalledDistribution(
   async function compileAndRunQuery(
     quickEval: boolean,
     selectedQuery: Uri | undefined,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<void> {
     if (qs !== undefined) {
@@ -491,10 +492,10 @@ async function activateWithInstalledDistribution(
 
   logger.log('Registering top-level command palette commands.');
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.runQuery',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
       ) => await compileAndRunQuery(false, uri, progress, token),
@@ -505,10 +506,10 @@ async function activateWithInstalledDistribution(
     )
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.runQueries',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         _: Uri | undefined,
         multi: Uri[]
@@ -533,7 +534,7 @@ async function activateWithInstalledDistribution(
 
         // Use a wrapped progress so that messages appear with the queries remaining in it.
         let queriesRemaining = queryUris.length;
-        function wrappedProgress(update: helpers.ProgressUpdate) {
+        function wrappedProgress(update: ProgressUpdate) {
           const message = queriesRemaining > 1
             ? `${queriesRemaining} remaining. ${update.message}`
             : update.message;
@@ -569,10 +570,10 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.quickEval',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
       ) => await compileAndRunQuery(true, uri, progress, token),
@@ -582,8 +583,8 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.quickQuery', async (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.quickQuery', async (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       displayQuickQuery(ctx, cliServer, databaseUI, progress, token),
@@ -594,7 +595,7 @@ async function activateWithInstalledDistribution(
   );
 
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.restartQueryServer', async () => {
+    commandRunner('codeQL.restartQueryServer', async () => {
       await qs.restartQueryServer();
       helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
@@ -602,24 +603,24 @@ async function activateWithInstalledDistribution(
     })
   );
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.chooseDatabaseFolder', (
-      progress: helpers.ProgressCallback,
+    commandRunner('codeQL.chooseDatabaseFolder', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseFolder(progress, token)
     )
   );
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.chooseDatabaseArchive', (
-      progress: helpers.ProgressCallback,
+    commandRunner('codeQL.chooseDatabaseArchive', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseArchive(progress, token)
     )
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseLgtm(progress, token),
@@ -628,8 +629,8 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseInternet(progress, token),
@@ -656,8 +657,8 @@ async function activateWithInstalledDistribution(
 
   const astViewer = new AstViewer();
   ctx.subscriptions.push(astViewer);
-  ctx.subscriptions.push(helpers.commandRunnerWithProgress('codeQL.viewAst', async (
-    progress: helpers.ProgressCallback,
+  ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
+    progress: ProgressCallback,
     token: CancellationToken
   ) => {
     const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)

extensions/ql-vscode/src/helpers.ts

@@ -3,195 +3,31 @@ import * as glob from 'glob-promise';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
 import {
-  CancellationToken,
   ExtensionContext,
-  ProgressOptions,
   window as Window,
-  workspace,
-  commands,
-  Disposable,
-  ProgressLocation
+  workspace
 } from 'vscode';
 import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
 
-export class UserCancellationException extends Error {
-  /**
-   * @param message The error message
-   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
-   */
-  constructor(message?: string, public readonly silent = false) {
-    super(message);
-  }
-}
-
-export interface ProgressUpdate {
-  /**
-   * The current step
-   */
-  step: number;
-  /**
-   * The maximum step. This *should* be constant for a single job.
-   */
-  maxStep: number;
-  /**
-   * The current progress message
-   */
-  message: string;
-}
-
-export type ProgressCallback = (p: ProgressUpdate) => void;
-
-/**
- * A task that handles command invocations from `commandRunner`
- * and includes a progress monitor.
- *
- *
- * Arguments passed to the command handler are passed along,
- * untouched to this `ProgressTask` instance.
- *
- * @param progress a progress handler function. Call this
- * function with a `ProgressUpdate` instance in order to
- * denote some progress being achieved on this task.
- * @param token a cencellation token
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-export type ProgressTask<R> = (
-  progress: ProgressCallback,
-  token: CancellationToken,
-  ...args: any[]
-) => Thenable<R>;
-
-/**
- * A task that handles command invocations from `commandRunner`.
- * Arguments passed to the command handler are passed along,
- * untouched to this `NoProgressTask` instance.
- *
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-type NoProgressTask = ((...args: any[]) => Promise<any>);
-
-/**
- * This mediates between the kind of progress callbacks we want to
- * write (where we *set* current progress position and give
- * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%).
- *
- * Where possible, the `commandRunner` function below should be used
- * instead of this function. The commandRunner is meant for wrapping
- * top-level commands and provides error handling and other support
- * automatically.
- *
- * Only use this function if you need a progress monitor and the
- * control flow does not always come from a command (eg- during
- * extension activation, or from an internal language server
- * request).
- */
-export function withProgress<R>(
-  options: ProgressOptions,
-  task: ProgressTask<R>,
-  ...args: any[]
-): Thenable<R> {
-  let progressAchieved = 0;
-  return Window.withProgress(options,
-    (progress, token) => {
-      return task(p => {
-        const { message, step, maxStep } = p;
-        const increment = 100 * (step - progressAchieved) / maxStep;
-        progressAchieved = step;
-        progress.report({ message, increment });
-      }, token, ...args);
-    });
-}
-
-/**
- * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
- *
- * In this variant of the command runner, no progress monitor is used.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task.
- */
-export function commandRunner(
-  commandId: string,
-  task: NoProgressTask,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    try {
-      return await task(...args);
-    } catch (e) {
-      const errorMessage = `${e.message || e} (${commandId})`;
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          logger.log(errorMessage);
-        } else {
-          showAndLogWarningMessage(errorMessage);
-        }
-      } else {
-        showAndLogErrorMessage(errorMessage);
-      }
-      return undefined;
-    }
-  });
-}
-
-/**
- * A generic wrapper for command registration.  This wrapper adds uniform error handling,
- * progress monitoring, and cancellation for commands.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task after the progress callback
- * and cancellation token.
- * @param progressOptions Progress options to be sent to the progress monitor.
- */
-export function commandRunnerWithProgress<R>(
-  commandId: string,
-  task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const progressOptionsWithDefaults = {
-      location: ProgressLocation.Notification,
-      ...progressOptions
-    };
-    try {
-      return await withProgress(progressOptionsWithDefaults, task, ...args);
-    } catch (e) {
-      const errorMessage = `${e.message || e} (${commandId})`;
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          logger.log(errorMessage);
-        } else {
-          showAndLogWarningMessage(errorMessage);
-        }
-      } else {
-        showAndLogErrorMessage(errorMessage);
-      }
-      return undefined;
-    }
-  });
-}
-
 /**
  * Show an error message and log it to the console
  *
  * @param message The message to show.
  * @param options.outputLogger The output logger that will receive the message
  * @param options.items A set of items that will be rendered as actions in the message.
+ * @param options.fullMessage An alternate message that is added to the log, but not displayed
+ *                           in the popup. This is useful for adding extra detail to the logs
+ *                           that would be too noisy for the popup.
  *
  * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
 export async function showAndLogErrorMessage(message: string, {
   outputLogger = logger,
-  items = [] as string[]
+  items = [] as string[],
+  fullMessage = undefined as (string | undefined)
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
+  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage, fullMessage);
 }
 /**
  * Show a warning message and log it to the console
@@ -226,10 +62,15 @@ export async function showAndLogInformationMessage(message: string, {
 
 type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
 
-async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
-  fn: ShowMessageFn): Promise<string | undefined> {
+async function internalShowAndLog(
+  message: string,
+  items: string[],
+  outputLogger = logger,
+  fn: ShowMessageFn,
+  fullMessage?: string
+): Promise<string | undefined> {
   const label = 'Show Log';
-  outputLogger.log(message);
+  outputLogger.log(fullMessage || message);
   const result = await fn(message, label, ...items);
   if (result === label) {
     outputLogger.show();
@@ -532,46 +373,3 @@ export async function isLikelyDatabaseRoot(maybeRoot: string) {
 export function isLikelyDbLanguageFolder(dbPath: string) {
   return !!path.basename(dbPath).startsWith('db-');
 }
-
-
-/**
- * Displays a progress monitor that indicates how much progess has been made
- * reading from a stream.
- *
- * @param readable The stream to read progress from
- * @param messagePrefix A prefix for displaying the message
- * @param totalNumBytes Total number of bytes in this stream
- * @param progress The progress callback used to set messages
- */
-export function reportStreamProgress(
-  readable: NodeJS.ReadableStream,
-  messagePrefix: string,
-  totalNumBytes?: number,
-  progress?: ProgressCallback
-) {
-  if (progress && totalNumBytes) {
-    let numBytesDownloaded = 0;
-    const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-    const updateProgress = () => {
-      progress({
-        step: numBytesDownloaded,
-        maxStep: totalNumBytes,
-        message: `${messagePrefix} [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
-      });
-    };
-
-    // Display the progress straight away rather than waiting for the first chunk.
-    updateProgress();
-
-    readable.on('data', data => {
-      numBytesDownloaded += data.length;
-      updateProgress();
-    });
-  } else if (progress) {
-    progress({
-      step: 1,
-      maxStep: 2,
-      message: `${messagePrefix} (Size unknown)`,
-    });
-  }
-}

extensions/ql-vscode/src/interface.ts

@@ -30,7 +30,7 @@ import {
   RawResultsSortState,
 } from './pure/interface-types';
 import { Logger } from './logging';
-import { commandRunner } from './helpers';
+import { commandRunner } from './commandRunner';
 import * as messages from './pure/messages';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';

extensions/ql-vscode/src/pure/messages.ts

@@ -385,8 +385,8 @@ export namespace ResultColumnKind {
    */
   export const BOOLEAN = 3;
   /**
- * A column of type `date`
- */
+   * A column of type `date`
+   */
   export const DATE = 4;
   /**
    * A column of a non-primitive type
@@ -406,6 +406,11 @@ export interface CompileUpgradeParams {
    * A directory to store parts of the compiled upgrade
    */
   upgradeTempDir: string;
+  /**
+   * Enable single file upgrades, set to true to allow
+   * using single file upgrades.
+   */
+  singleFileUpgrades: true;
 }
 
 /**
@@ -492,10 +497,13 @@ export interface UpgradeDescription {
   newSha: string;
 }
 
+
+export type CompiledUpgrades = MultiFileCompiledUpgrades | SingleFileCompiledUpgrade
+
 /**
- * A compiled upgrade.
+ * The parts shared by all compiled upgrades
  */
-export interface CompiledUpgrades {
+interface CompiledUpgradesBase {
   /**
   * The initial sha of the dbscheme to upgrade from
   */
@@ -504,14 +512,46 @@ export interface CompiledUpgrades {
    * The path to the new dataset statistics
    */
   newStatsPath: string;
+  /**
+   * The sha of the target dataset.
+   */
+  targetSha: string;
+}
+
+
+/**
+ * A compiled upgrade.
+ * The upgrade is spread among multiple files.
+ */
+interface MultiFileCompiledUpgrades extends CompiledUpgradesBase {
+  /**
+   * The path to the new dataset dbscheme
+   */
+  newDbscheme: string;
   /**
    * The steps in the upgrade path
    */
   scripts: CompiledUpgradeScript[];
   /**
-   * The sha of the target dataset.
+   * Will never exist in an old result
    */
-  targetSha: string;
+  compiledUpgradeFile?: never;
+}
+
+
+/**
+ * A compiled upgrade.
+ * The upgrade is in a single file.
+ */
+export interface SingleFileCompiledUpgrade extends CompiledUpgradesBase {
+  /**
+   * The steps in the upgrade path
+   */
+  descriptions: UpgradeDescription[];
+  /**
+   * A path to a file containing the upgrade
+   */
+  compiledUpgradeFile: string;
 }
 
 /**

extensions/ql-vscode/src/query-history.ts

@@ -4,7 +4,15 @@ import { window as Window } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
-import * as helpers from './helpers';
+import {
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+  showAndLogWarningMessage,
+  showBinaryChoiceDialog
+} from './helpers';
+import {
+  commandRunner
+} from './commandRunner';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
@@ -207,55 +215,55 @@ export class QueryHistoryManager extends DisposableObject {
 
     logger.log('Registering query history panel commands.');
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.openQuery',
         this.handleOpenQuery.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.removeHistoryItem',
         this.handleRemoveHistoryItem.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.setLabel',
         this.handleSetLabel.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.compareWith',
         this.handleCompareWith.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.showQueryLog',
         this.handleShowQueryLog.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.showQueryText',
         this.handleShowQueryText.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.viewSarif',
         this.handleViewSarif.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.viewDil',
         this.handleViewDil.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.itemClicked',
         async (item: CompletedQuery) => {
           return this.handleItemClicked(item, [item]);
@@ -376,7 +384,7 @@ export class QueryHistoryManager extends DisposableObject {
         this.doCompareCallback(from, to);
       }
     } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+      showAndLogErrorMessage(e.message);
     }
   }
 
@@ -423,7 +431,7 @@ export class QueryHistoryManager extends DisposableObject {
     if (singleItem.logFileLocation) {
       await this.tryOpenExternalFile(singleItem.logFileLocation);
     } else {
-      helpers.showAndLogWarningMessage('No log file available');
+      showAndLogWarningMessage('No log file available');
     }
   }
 
@@ -468,7 +476,7 @@ export class QueryHistoryManager extends DisposableObject {
       );
     } else {
       const label = singleItem.getLabel();
-      helpers.showAndLogInformationMessage(
+      showAndLogInformationMessage(
         `Query ${label} has no interpreted results.`
       );
     }
@@ -547,7 +555,7 @@ export class QueryHistoryManager extends DisposableObject {
         ) ||
         e.message.includes('too large to open')
       ) {
-        const res = await helpers.showBinaryChoiceDialog(
+        const res = await showBinaryChoiceDialog(
           `VS Code does not allow extensions to open files >50MB. This file
 exceeds that limit. Do you want to open it outside of VS Code?
 
@@ -558,11 +566,11 @@ the file in the file explorer and dragging it into the workspace.`
           try {
             await vscode.commands.executeCommand('revealFileInOS', uri);
           } catch (e) {
-            helpers.showAndLogErrorMessage(e.message);
+            showAndLogErrorMessage(e.message);
           }
         }
       } else {
-        helpers.showAndLogErrorMessage(`Could not open file ${fileLocation}`);
+        showAndLogErrorMessage(`Could not open file ${fileLocation}`);
         logger.log(e.message);
         logger.log(e.stack);
       }
@@ -616,7 +624,7 @@ the file in the file explorer and dragging it into the workspace.`
 
   private assertSingleQuery(multiSelect: CompletedQuery[] = [], message = 'Please select a single query.') {
     if (multiSelect.length > 1) {
-      helpers.showAndLogErrorMessage(
+      showAndLogErrorMessage(
         message
       );
       return false;

extensions/ql-vscode/src/quick-query.ts

@@ -10,11 +10,13 @@ import {
   getInitialQueryContents,
   getPrimaryDbscheme,
   getQlPackForDbscheme,
-  ProgressCallback,
   showAndLogErrorMessage,
   showBinaryChoiceDialog,
-  UserCancellationException
 } from './helpers';
+import {
+  ProgressCallback,
+  UserCancellationException
+} from './commandRunner';
 
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';

extensions/ql-vscode/src/run-queries.ts

@@ -15,7 +15,8 @@ import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import * as cli from './cli';
 import * as config from './config';
 import { DatabaseItem, getUpgradesDirectories } from './databases';
-import * as helpers from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { DatabaseInfo, QueryMetadata, ResultsPaths } from './pure/interface-types';
 import { logger } from './logging';
 import * as messages from './pure/messages';
@@ -79,7 +80,7 @@ export class QueryInfo {
 
   async run(
     qs: qsClient.QueryServerClient,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.EvaluationResult> {
     let result: messages.EvaluationResult | null = null;
@@ -121,7 +122,7 @@ export class QueryInfo {
 
   async compile(
     qs: qsClient.QueryServerClient,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.CompilationMessage[]> {
     let compiled: messages.CheckQueryResult | undefined;
@@ -209,7 +210,7 @@ export interface QueryWithResults {
 export async function clearCacheInDatabase(
   qs: qsClient.QueryServerClient,
   dbItem: DatabaseItem,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<messages.ClearCacheResult> {
   if (dbItem.contents === undefined) {
@@ -285,10 +286,10 @@ async function checkDbschemeCompatibility(
   cliServer: cli.CodeQLCliServer,
   qs: qsClient.QueryServerClient,
   query: QueryInfo,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<void> {
-  const searchPath = helpers.getOnDiskWorkspaceFolders();
+  const searchPath = getOnDiskWorkspaceFolders();
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
     const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
@@ -364,7 +365,7 @@ async function promptUserToSaveChanges(document: TextDocument): Promise<boolean>
       }
 
       if (chosenItem === cancelItem) {
-        throw new helpers.UserCancellationException('Query run cancelled.', true);
+        throw new UserCancellationException('Query run cancelled.', true);
       }
     }
   }
@@ -454,7 +455,7 @@ export async function compileAndRunQueryAgainstDatabase(
   db: DatabaseItem,
   quickEval: boolean,
   selectedQueryUri: Uri | undefined,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
   templates?: messages.TemplateDefinitions,
 ): Promise<QueryWithResults> {
@@ -474,7 +475,7 @@ export async function compileAndRunQueryAgainstDatabase(
   }
 
   // Get the workspace folder paths.
-  const diskWorkspaceFolders = helpers.getOnDiskWorkspaceFolders();
+  const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
   // Figure out the library path for the query.
   const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, queryPath);
 
@@ -533,7 +534,7 @@ export async function compileAndRunQueryAgainstDatabase(
     if (result.resultType !== messages.QueryResultType.SUCCESS) {
       const message = result.message || 'Failed to run query';
       logger.log(message);
-      helpers.showAndLogErrorMessage(message);
+      showAndLogErrorMessage(message);
     }
     return {
       query,
@@ -564,9 +565,9 @@ export async function compileAndRunQueryAgainstDatabase(
       qs.logger.log(formatted);
     }
     if (quickEval && formattedMessages.length <= 3) {
-      helpers.showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
+      showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
     } else {
-      helpers.showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
+      showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
         ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
         ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
         ' and choose CodeQL Query Server from the dropdown.');

extensions/ql-vscode/src/test-adapter.ts

@@ -220,14 +220,13 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
       const state = event.pass
         ? 'passed'
         : event.messages?.length
-        ? 'errored'
-        : 'failed';
+          ? 'errored'
+          : 'failed';
       let message: string | undefined;
       if (event.diff?.length) {
         message = ['', `${state}: ${event.test}`, ...event.diff, ''].join('\n');
         testLogger.log(message);
       }
-      (event.diff || []).join('\n');
       this._testStates.fire({
         type: 'test',
         state,

extensions/ql-vscode/src/upgrades.ts

@@ -1,6 +1,7 @@
 import * as vscode from 'vscode';
 import { DatabaseItem } from './databases';
-import * as helpers from './helpers';
+import { showAndLogErrorMessage } from './helpers';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 import * as messages from './pure/messages';
 import * as qsClient from './queryserver-client';
@@ -24,7 +25,7 @@ async function checkAndConfirmDatabaseUpgrade(
   db: DatabaseItem,
   targetDbScheme: vscode.Uri,
   upgradesDirectories: vscode.Uri[],
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.UpgradeParams | undefined> {
   if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
@@ -75,7 +76,7 @@ async function checkAndConfirmDatabaseUpgrade(
   if (curSha != targetSha) {
     // Newlines aren't rendered in notifications: https://github.com/microsoft/vscode/issues/48900
     // A modal dialog would be rendered better, but is more intrusive.
-    await helpers.showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
+    await showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
     Can upgrade from ${checkedUpgrades.initialSha} (current) to ${curSha}, but cannot reach ${targetSha} (target).`);
     // TODO: give a more informative message if we think the DB is ahead of the target DB scheme
     return;
@@ -114,7 +115,7 @@ async function checkAndConfirmDatabaseUpgrade(
     return params;
   }
   else {
-    throw new helpers.UserCancellationException('User cancelled the database upgrade.');
+    throw new UserCancellationException('User cancelled the database upgrade.');
   }
 }
 
@@ -128,7 +129,7 @@ export async function upgradeDatabase(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem, targetDbScheme: vscode.Uri,
   upgradesDirectories: vscode.Uri[],
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.RunUpgradeResult | undefined> {
   const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories, progress, token);
@@ -142,7 +143,7 @@ export async function upgradeDatabase(
     compileUpgradeResult = await compileDatabaseUpgrade(qs, upgradeParams, progress, token);
   }
   catch (e) {
-    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
+    showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
     return;
   }
   finally {
@@ -151,17 +152,24 @@ export async function upgradeDatabase(
 
   if (compileUpgradeResult.compiledUpgrades === undefined) {
     const error = compileUpgradeResult.error || '[no error message available]';
-    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
+    (`Compilation of database upgrades failed: ${error}`);
     return;
   }
 
   try {
     qs.logger.log('Running the following database upgrade:');
-    qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
+    // We use the presence of compiledUpgradeFile to check
+    // if it is multifile or not. We need to explicitly check undefined
+    // as the types claim the empty string is a valid value
+    if (compileUpgradeResult.compiledUpgrades.compiledUpgradeFile === undefined) {
+      qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
+    } else {
+      qs.logger.log(compileUpgradeResult.compiledUpgrades.descriptions.map(s => s.description).join('\n'));
+    }
     return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
   }
   catch (e) {
-    helpers.showAndLogErrorMessage(`Database upgrade failed: ${e}`);
+    showAndLogErrorMessage(`Database upgrade failed: ${e}`);
     return;
   }
   finally {
@@ -172,7 +180,7 @@ export async function upgradeDatabase(
 async function checkDatabaseUpgrade(
   qs: qsClient.QueryServerClient,
   upgradeParams: messages.UpgradeParams,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.CheckUpgradeResult> {
   progress({
@@ -187,12 +195,13 @@ async function checkDatabaseUpgrade(
 async function compileDatabaseUpgrade(
   qs: qsClient.QueryServerClient,
   upgradeParams: messages.UpgradeParams,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.CompileUpgradeResult> {
   const params: messages.CompileUpgradeParams = {
     upgrade: upgradeParams,
-    upgradeTempDir: upgradesTmpDir.name
+    upgradeTempDir: upgradesTmpDir.name,
+    singleFileUpgrades: true
   };
 
   progress({
@@ -208,7 +217,7 @@ async function runDatabaseUpgrade(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
   upgrades: messages.CompiledUpgrades,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.RunUpgradeResult> {
 

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -7,7 +7,7 @@ import { extensions, CancellationToken, Uri, window } from 'vscode';
 import { CodeQLExtensionInterface } from '../../extension';
 import { DatabaseManager } from '../../databases';
 import { promptImportLgtmDatabase, importArchiveDatabase, promptImportInternetDatabase } from '../../databaseFetcher';
-import { ProgressCallback } from '../../helpers';
+import { ProgressCallback } from '../../commandRunner';
 import { dbLoc, DB_URL, storagePath } from './global.helper';
 
 /**

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -40,7 +40,7 @@ describe('Use cli', function() {
     ]);
   });
 
-  it.only('should resolve query packs', async function() {
+  it('should resolve query packs', async function() {
     skipIfNoCodeQL(this);
     const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
     // should have a bunch of qlpacks. just check that a few known ones exist

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -72,7 +72,7 @@ export async function ensureCli(useCli: boolean) {
       console.log('Total content size', Math.round(contentLength / _1MB), 'MB');
       const archiveFile = fs.createWriteStream(downloadedFilePath);
       const body = assetStream.body;
-      await new Promise((resolve, reject) => {
+      await new Promise<void>((resolve, reject) => {
         let numBytesDownloaded = 0;
         let lastMessage = 0;
         body.on('data', (data) => {
@@ -117,7 +117,11 @@ function hasCodeQL() {
 
 export function skipIfNoCodeQL(context: Mocha.Context) {
   if (!hasCodeQL()) {
-    console.log('The CodeQL libraries are not available as a folder in this workspace. To fix: checkout the github/codeql repository and set the TEST_CODEQL_PATH environment variable to the checked out directory.');
+    console.log([
+      'The CodeQL libraries are not available as a folder in this workspace.',
+      'To fix in CI: checkout the github/codeql repository and set the \'TEST_CODEQL_PATH\' environment variable to the checked out directory.',
+      'To fix when running from vs code, see the comment in the launch.json file in the \'Launch Integration Tests - With CLI\' section.'
+    ].join('\n\n'));
     context.skip();
   }
 }

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -17,7 +17,7 @@ import {
 import { Logger } from '../../logging';
 import { QueryServerClient } from '../../queryserver-client';
 import { registerDatabases } from '../../pure/messages';
-import { ProgressCallback } from '../../helpers';
+import { ProgressCallback } from '../../commandRunner';
 import { CodeQLCliServer } from '../../cli';
 import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
 

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -214,7 +214,6 @@ describe('Launcher path', () => {
 
   it('should warn when using a hard-coded deprecated launcher name', async () => {
     launcherThatExists = 'codeql.cmd';
-    path.sep;
     const result = await getExecutableFromDirectory('abc');
     expect(fsSpy).to.have.been.calledWith(pathToExe);
     expect(fsSpy).to.have.been.calledWith(pathToCmd);

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -7,7 +7,8 @@ import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as sinon from 'sinon';
 
-import { getInitialQueryContents, InvocationRateLimiter, isLikelyDbLanguageFolder, reportStreamProgress } from '../../helpers';
+import { getInitialQueryContents, InvocationRateLimiter, isLikelyDbLanguageFolder } from '../../helpers';
+import { reportStreamProgress } from '../../commandRunner';
 
 describe('helpers', () => {
   let sandbox: sinon.SinonSandbox;

extensions/ql-vscode/src/vscode-utils/ui-service.ts

@@ -1,6 +1,6 @@
 import { TreeDataProvider, window } from 'vscode';
 import { DisposableObject } from './disposable-object';
-import { commandRunner } from '../helpers';
+import { commandRunner } from '../commandRunner';
 
 /**
  * A VS Code service that interacts with the UI, including handling commands.