v1.3.6 (#671)

This will prevent selections jumping around when an ast entry is
selected and its child has the same source location as the current
selection.

.editorconfig

@@ -4,3 +4,4 @@ indent_size = 2
 charset = utf-8
 end_of_line = lf
 trim_trailing_whitespace = true
+insert_final_newline = true

.gitattributes

@@ -12,3 +12,10 @@ yarn.lock                    merge=binary
 # For more information, see this issue: https://github.com/Microsoft/web-build-tools/issues/1088
 #
 *.json                       linguist-language=JSON-with-Comments
+
+# Reduce incidence of needless merge conflicts on CHANGELOG.md
+# The man page at
+# https://mirrors.edge.kernel.org/pub/software/scm/git/docs/gitattributes.html
+# suggests that this might interleave lines arbitrarily, but empirically
+# it keeps added chunks contiguous
+CHANGELOG.md                merge=union

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,20 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new-extension-release.md

@@ -0,0 +1,18 @@
+---
+name: New extension release
+about: Create an issue with a checklist for the release steps (write access required
+  for the steps)
+title: Release Checklist for version xx.xx.xx
+labels: ''
+assignees: ''
+
+---
+
+- [ ] Update this issue title to refer to the version of the release
+- [ ] Trigger a release build on Actions by adding a new tag on branch `main` of the format `vxx.xx.xx`
+- [ ] Monitor the status of the release build in the `Release` workflow in the Actions tab.
+- [ ] Download the VSIX from the draft GitHub release that is created when the release build finishes.
+- [ ] Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
+- [ ] Click the `...` menu in the CodeQL row and click **Update**.
+- [ ] Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
+- [ ] Publish the draft GitHub release and confirm the new release is marked as the latest release at https://github.com/github/vscode-codeql/releases.

.github/codeql/codeql-config.yml

@@ -0,0 +1,12 @@
+name: "CodeQL config"
+queries:
+  - name: Run standard queries
+    uses: security-and-quality
+  - name: Run custom javascript queries
+    uses: ./.github/codeql/queries
+paths:
+  - ./extensions/ql-vscode
+paths-ignore:
+  - '**/node_modules'
+  - '**/build'
+  - '**/out'

.github/codeql/queries/assert-pure.ql

@@ -0,0 +1,21 @@
+/**
+ * @name Unwanted dependency on vscode API
+ * @kind problem
+ * @problem.severity error
+ * @id vscode-codeql/assert-pure
+ * @description The modules stored under `pure` and tested in the `pure-tests`
+ * are intended to be "pure".
+ */
+import javascript
+
+class VSCodeImport extends ASTNode {
+  VSCodeImport() {
+    this.(Import).getImportedPath().getValue() = "vscode"
+  }
+}
+
+from Module m, VSCodeImport v
+where
+  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
+  m.getAnImportedModule*().getAnImport() = v
+select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/qlpack.yml

@@ -0,0 +1,3 @@
+name: vscode-codeql-custom-queries-javascript
+version: 0.0.0
+libraryPathDependencies: codeql-javascript

.github/pull_request_template.md

@@ -0,0 +1,12 @@
+<!-- Thank you for submitting a pull request. Please read our pull request guidelines before
+  submitting your pull request:
+  https://github.com/github/vscode-codeql/blob/main/CONTRIBUTING.md#submitting-a-pull-request.
+-->
+
+Replace this with a description of the changes your pull request makes.
+
+## Checklist
+
+- [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
+- [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
+- [ ] `@github/docs-content-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -0,0 +1,25 @@
+name: "Code Scanning - CodeQL"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  codeql:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        with:
+            languages: javascript
+            config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

.github/workflows/label-issue.yml

@@ -0,0 +1,15 @@
+name: Label issue
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  label:
+    name: Label issue
+    runs-on: ubuntu-latest
+    steps:
+      - name: Label issue
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo '{"labels": ["VSCode"]}' | gh api repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels --input -

.github/workflows/main.yml

@@ -1,5 +1,9 @@
 name: Build Extension
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   build:
@@ -10,15 +14,24 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.18.1'
+
+      - name: Install dependencies
+        run: |
+          cd extensions/ql-vscode
+          npm install
+        shell: bash
+
       - name: Build
         run: |
-          cd build
-          npm install
-          npm run build-ci
+          cd extensions/ql-vscode
+          npm run build
         shell: bash
 
       - name: Prepare artifacts
@@ -28,7 +41,7 @@ jobs:
           cp dist/*.vsix artifacts
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@master
+        uses: actions/upload-artifact@v2
         if: matrix.os == 'ubuntu-latest'
         with:
           name: vscode-codeql-extension
@@ -42,21 +55,52 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
-      # We have to build the dependencies in `lib` before running any tests.
-      - name: Build
-        run: |
-          cd build
-          npm install
-          npm run build-ci
-        shell: bash
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.18.1'
 
-      - name: Run unit tests
+      # We have to build the dependencies in `lib` before running any tests.
+      - name: Install dependencies
         run: |
           cd extensions/ql-vscode
+          npm install
+        shell: bash
+
+      - name: Build
+        run: |
+          cd extensions/ql-vscode
+          npm run build
+        shell: bash
+
+      - name: Lint
+        run: |
+          cd extensions/ql-vscode
+          npm run lint
+
+      - name: Install CodeQL
+        run: |
+          mkdir codeql-home
+          curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
+          unzip -q -o codeql-home/codeql.zip -d codeql-home
+          unzip -q -o codeql-home/codeql.zip codeql/codeql.exe -d codeql-home
+          rm codeql-home/codeql.zip
+        shell: bash
+
+      - name: Run unit tests (Linux)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          cd extensions/ql-vscode
+          CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
+
+      - name: Run unit tests (Windows)
+        if: matrix.os == 'windows-latest'
+        run: |
+          cd extensions/ql-vscode
+          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
@@ -70,4 +114,4 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: |
           cd extensions/ql-vscode
-          npm run integration
+          npm run integration

.github/workflows/release.yml

@@ -27,14 +27,22 @@ jobs:
     # TODO Share steps with the main workflow.
     steps:
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.18.1'
+
+      - name: Install dependencies
+        run: |
+          cd extensions/ql-vscode
+          npm install
+        shell: bash
 
       - name: Build
         run: |
-          cd build
-          npm install
-          # Release build instead of dev build.
-          npm run build-release
+          cd extensions/ql-vscode
+          npm run build -- --release
         shell: bash
 
       - name: Prepare artifacts
@@ -47,15 +55,15 @@ jobs:
           VSIX_PATH="$(ls dist/*.vsix)"
           echo "::set-output name=vsix_path::$VSIX_PATH"
           # Transform the GitHub ref so it can be used in a filename.
-          # This is mainly needed for testing branches that modify this workflow.
-          REF_NAME="$(echo ${{ github.ref }} | sed -e 's:/:-:g')"
+          # The last sed invocation is used for testing branches that modify this workflow.
+          REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
       # Uploading artifacts is not necessary to create a release.
       # This is just in case the release itself fails and we want to access the built artifacts from Actions.
       # TODO Remove if not useful.
       - name: Upload artifacts
-        uses: actions/upload-artifact@master
+        uses: actions/upload-artifact@v2
         with:
           name: vscode-codeql-extension
           path: artifacts
@@ -89,6 +97,13 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
+      # The checkout action does not fetch the main branch.
+      # Fetch the main branch so that we can base the version bump PR against main.
+      - name: Fetch main branch
+        run: |
+          git fetch --depth=1 origin main:main
+          git checkout main
+
       - name: Bump patch version
         id: bump-patch-version
         if: success()
@@ -99,8 +114,14 @@ jobs:
           NEXT_VERSION="$(npm version patch)"
           echo "::set-output name=next_version::$NEXT_VERSION"
 
+      - name: Add changelog for next release
+        if: success()
+        run: |
+          cd extensions/ql-vscode
+          perl -i -pe 's/^/## \[UNRELEASED\]\n\n/ if($.==3)' CHANGELOG.md
+
       - name: Create version bump PR
-        uses: peter-evans/create-pull-request@7531167f24e3914996c8d5110b5e08478ddadff9  # v1.8.0
+        uses: peter-evans/create-pull-request@c7f493a8000b8aeb17a1332e326ba76b57cb83eb # v3.4.1
         if: success()
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -108,4 +129,4 @@ jobs:
           title: Bump version to ${{ steps.bump-patch-version.outputs.next_version }}
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
-          branch-suffix: none
+          base: main

.gitignore

@@ -18,3 +18,4 @@ artifacts/
 # Rush files
 /common/temp/**
 package-deps.json
+**/.rush/temp

.vscode/extensions.json

@@ -3,8 +3,10 @@
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
-    "eamodio.tsl-problem-matcher"
+    "eamodio.tsl-problem-matcher",
+    "dbaeumer.vscode-eslint",
+    "eternalphane.tsfmt-vscode"
   ],
   // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
   "unwantedRecommendations": []
-}
+}

.vscode/launch.json

@@ -8,41 +8,43 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--disable-extensions"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-bqrs/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io-node/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-vscode-utils/out/**/*.js"
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
-      "preLaunchTask": "Build"
+      "env": {
+        // uncomment to allow debugging the language server Java process from a remote java debugger
+        // "DEBUG_LANGUAGE_SERVER": "true"
+      }
     },
     {
       "name": "Launch Unit Tests (vscode-codeql)",
-      "type": "extensionHost",
+      "type": "node",
       "request": "launch",
-      "runtimeExecutable": "${execPath}",
-      "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/test",
-        "--disable-extensions"
+      "program": "${workspaceFolder}/extensions/ql-vscode/node_modules/mocha/bin/_mocha",
+      "showAsyncStacks": true,
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
+      "runtimeArgs": [
+        "--inspect=9229"
       ],
+      "args": [
+        "--exit",
+        "-u",
+        "bdd",
+        "--colors",
+        "--diff",
+        "-r",
+        "ts-node/register",
+        "test/pure-tests/**/*.ts"
+      ],
+      "port": 9229,
       "stopOnEntry": false,
       "sourceMaps": true,
-      "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-bqrs/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io-node/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-vscode-utils/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/test/**/*.js"
-      ],
-      "preLaunchTask": "Build"
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
     },
     {
       "name": "Launch Integration Tests - No Workspace (vscode-codeql)",
@@ -50,17 +52,14 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
-        "--disable-extensions"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/**/*.js"
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
-      "preLaunchTask": "Build"
     },
     {
       "name": "Launch Integration Tests - Minimal Workspace (vscode-codeql)",
@@ -68,17 +67,15 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
-        "${workspaceRoot}/extensions/ql-vscode/test/data",
+        "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/**/*.js"
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
-      "preLaunchTask": "Build"
     }
   ]
-}
+}

.vscode/settings.json

@@ -1,14 +1,40 @@
 // Place your settings in this file to overwrite default and user settings.
 {
   "files.exclude": {
-    "out": false // set this to true to hide the "out" folder with the compiled JS files
+    "**/out": true, // set this to true to hide the "out" folder with the compiled JS files
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
   "files.watcherExclude": {
     "**/.git/**": true,
-    "**/node_modules/*/**": true
+    "**/out": true,
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
   "search.exclude": {
-    "out": true // set this to false to include "out" folder in search results
+    "**/out": true, // set this to false to include "out" folder in search results
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
-  "typescript.tsdk": "./common/temp/node_modules/typescript/lib" // we want to use the TS server from our node_modules folder to control its version
-}
+  "typescript.tsdk": "./common/temp/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
+  "eslint.validate": [
+    "javascript",
+    "javascriptreact",
+    "typescript",
+    "typescriptreact"
+  ],
+  "eslint.options": {
+    // This is necessary so that eslint can properly resolve its plugins
+    "resolvePluginsRelativeTo": "./extensions/ql-vscode"
+  },
+  "editor.formatOnSave": false,
+  "typescript.preferences.quoteStyle": "single",
+  "javascript.preferences.quoteStyle": "single",
+  "editor.wordWrapColumn": 100
+}

.vscode/tasks.json

@@ -10,34 +10,10 @@
         "kind": "build",
         "isDefault": true
       },
-      "command": "node common/scripts/install-run-rush.js build --verbose",
-      "presentation": {
-        "echo": true,
-        "reveal": "always",
-        "focus": false,
-        "panel": "shared",
-        "showReuseMessage": true,
-        "clear": true
+      "command": "npm run build",
+      "options": {
+        "cwd": "extensions/ql-vscode/"
       },
-      "problemMatcher": [
-        {
-          "owner": "typescript",
-          "fileLocation": "absolute",
-          "pattern": {
-            "regexp": "^\\[gulp-typescript\\] ([^(]+)\\((\\d+|\\d+,\\d+|\\d+,\\d+,\\d+,\\d+)\\): error TS\\d+: (.*)$",
-            "file": 1,
-            "location": 2,
-            "message": 3
-          },
-        },
-        "$ts-webpack"
-      ]
-    },
-    {
-      "label": "Rebuild",
-      "type": "shell",
-      "group": "build",
-      "command": "node common/scripts/install-run-rush.js rebuild --verbose",
       "presentation": {
         "echo": true,
         "reveal": "always",
@@ -56,41 +32,17 @@
             "location": 2,
             "message": 3
           }
-        }
+        },
+        "$ts-webpack"
       ]
     },
-    {
-      "label": "Update",
-      "type": "shell",
-      "command": "node common/scripts/install-run-rush.js update",
-      "presentation": {
-        "echo": true,
-        "reveal": "always",
-        "focus": false,
-        "panel": "shared",
-        "showReuseMessage": true,
-        "clear": true
-      },
-      "problemMatcher": []
-    },
-    {
-      "label": "Update (full)",
-      "type": "shell",
-      "command": "node common/scripts/install-run-rush.js update --full",
-      "presentation": {
-        "echo": true,
-        "reveal": "always",
-        "focus": false,
-        "panel": "shared",
-        "showReuseMessage": true,
-        "clear": true
-      },
-      "problemMatcher": []
-    },
     {
       "label": "Format",
       "type": "shell",
-      "command": "node common/scripts/install-run-rush.js format",
+      "command": "npm run format",
+      "options": {
+        "cwd": "extensions/ql-vscode/"
+      },
       "presentation": {
         "echo": true,
         "reveal": "always",
@@ -100,6 +52,15 @@
         "clear": true
       },
       "problemMatcher": []
+    },
+    {
+      "type": "npm",
+      "script": "watch",
+      "path": "extensions/ql-vscode/",
+      "problemMatcher": [
+        "$gulp-tsc"
+      ],
+      "group": "build"
     }
   ]
-}
+}

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-## Contributing
+# Contributing
 
 [fork]: https://github.com/github/vscode-codeql/fork
 [pr]: https://github.com/github/vscode-codeql/compare
@@ -13,94 +13,53 @@ Please note that this project is released with a [Contributor Code of Conduct][c
 
 ## Submitting a pull request
 
-0. [Fork][fork] and clone the repository
-0. Set up a local build
-0. Create a new branch: `git checkout -b my-branch-name`
-0. Make your change
-0. Push to your fork and [submit a pull request][pr]
-0. Pat yourself on the back and wait for your pull request to be reviewed and merged.
+1. [Fork][fork] and clone the repository
+1. Set up a local build
+1. Create a new branch: `git checkout -b my-branch-name`
+1. Make your change
+1. Push to your fork and [submit a pull request][pr]
+1. Pat yourself on the back and wait for your pull request to be reviewed and merged.
 
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 
-- Follow the [style guide][style].
-- Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
-- Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Follow the [style guide][style].
+* Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
+* Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
+* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
 Make sure you have a fairly recent version of vscode (>1.32) and are using nodejs
 version >=v10.13.0. (Tested on v10.15.1 and v10.16.0).
 
-This repo uses [Rush](https://rushjs.io) to handle package management, building, and other
-operations across multiple projects. See the Rush "[Getting started as a developer](https://rushjs.io/pages/developer/new_developer/)" docs
-for more details.
+### Installing all packages
 
-If you plan on building from the command line, it's easiest if Rush is installed globally:
+From the command line, go to the directory `extensions/ql-vscode` and run
 
 ```shell
-npm install -g @microsoft/rush
+npm install
 ```
 
-Note that when you run the `rush` command from the globally installed version, it will examine the
-`rushVersion` property in the repo's `rush.json`, and if it differs from the globally installed
-version, it will download, cache, and run the version of Rush specified in the `rushVersion`
-property.
+### Building the extension
 
-If you plan on only building via VS Code tasks, you don't need Rush installed at all, since those
-tasks run `common/scripts/install-run-rush.js` to bootstrap a locally installed and cached copy of
-Rush.
-
-### Building
-
-#### Installing all packages (instead of `npm install`)
-
-After updating any `package.json` file, or after checking or pulling a new branch, you need to
-make sure all the right npm packages are installed, which you would normally do via `npm install` in
-a single-project repo. With Rush, you need to do an "update" instead:
-
-##### From VS Code
-
-`Terminal > Run Task... > Update`
-
-##### From the command line
+From the command line, go to the directory `extensions/ql-vscode` and run
 
 ```shell
-$ rush update
+npm run build
+npm run watch
 ```
 
-#### Building all projects (instead of `gulp`)
+Alternatively, you can build the extension within VS Code via `Terminal > Run Build Task...` (or `Ctrl+Shift+B` with the default key bindings). And you can run the watch command via `Terminal > Run Task` and then select `npm watch` from the menu.
 
-Rush builds all projects in the repo, in dependency order, building multiple projects in parallel
-where possible. By default, the build also packages the extension itself into a .vsix file in the
-`dist` directory. To build:
+Before running any of the launch commands, be sure to have run the `build` command to ensure that the JavaScript is compiled and the resources are copied to the proper location.
 
-##### From VS Code
+We recommend that you keep `npm run watch` running in the backgound and you only need to re-run `npm run build` in the following situations:
 
-`Terminal > Run Build Task...` (or just `Ctrl+Shift+B` with the default key bindings)
+1. on first checkout
+2. whenever any of the non-TypeScript resources have changed
+3. on any change to files included in the webview
 
-##### From the command line
-
-```shell
-rush build --verbose
-```
-
-#### Forcing a clean build
-
-Rush does a reasonable job of detecting on its own which projects need to be rebuilt, but if you need to
-force a full rebuild of all projects:
-
-##### From VS Code
-
-`Terminal > Run Task... > Rebuild`
-
-##### From the command line
-
-```shell
-rush rebuild --verbose
-```
-
-### Installing
+### Installing the extension
 
 You can install the `.vsix` file from within VS Code itself, from the Extensions container in the sidebar:
 
@@ -118,18 +77,40 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
+### Running the unit/integration tests
+
+Ensure the `CODEQL_PATH` environment variable is set to point to the `codeql` cli executable.
+
+Outside of vscode, run:
+
+```shell
+npm run test && npm run integration
+```
+
+Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
+
 ## Releasing (write access required)
 
-1. Trigger a release build on Actions by adding a new tag on master of the format `vxx.xx.xx`
+1. Double-check the `CHANGELOG.md` contains all desired change comments
+   and has the version to be released with date at the top.
+1. Double-check that the extension `package.json` has the version you intend to release.
+   If you are doing a patch release (as opposed to minor or major version) this should already
+   be correct.
+1. Trigger a release build on Actions by adding a new tag on branch `main` of the format `vxx.xx.xx`
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
-1. Download the VSIX from the draft GitHub release that is created when the release build finishes.
+1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
+1. Optionally unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
+   or look at the source if there's any doubt the right code is being shipped.
 1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
 1. Click the `...` menu in the CodeQL row and click **Update**.
 1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Publish the GitHub release.
+1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
+1. If documentation changes need to be published, notify documentation team that release has been made.
+1. Review and merge the version bump PR that is automatically created by Actions.
 
 ## Resources
 
-- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
-- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
-- [GitHub Help](https://help.github.com)
+* [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+* [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+* [GitHub Help](https://help.github.com)

README.md

@@ -4,15 +4,16 @@ This project is an extension for Visual Studio Code that adds rich language supp
 
 The extension is released. You can download it from the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql).
 
-To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
+To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
-![CI status badge](https://github.com/github/vscode-codeql/workflows/Build%20Extension/badge.svg)
+[![CI status badge](https://github.com/github/vscode-codeql/workflows/Build%20Extension/badge.svg)](https://github.com/github/vscode-codeql/actions?query=workflow%3A%22Build+Extension%22+branch%3Amaster)
+[![VS Marketplace badge](https://vsmarketplacebadge.apphb.com/version/github.vscode-codeql.svg)](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql)
 
 ## Features
 
 * Enables you to use CodeQL to query databases and discover problems in codebases.
 * Shows the flow of data through the results of path queries, which is essential for triaging security results.
-* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/Semmle/ql).
+* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
 

build/README.md

@@ -1,12 +0,0 @@
-GitHub Actions Build directory
-===
-
-The point of this directory is to allow us to do a local installation *of* the rush
-tool, since 
- - installing globally is not permitted on github actions
- - installing locally in the root directory of the repo creates `node_modules` there,
-   and rush itself gives error messages since it thinks `node_modules` is not supposed 
-   to exist, since rush is supposed to be managing subproject dependencies.
-
-Running rush from a subdirectory searches parent directories for `rush.json`
-and does the build starting from that file's location.

build/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "build",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "dependencies": {
-    "@microsoft/rush": "^5.10.3"
-  },
-  "devDependencies": {},
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "build": "rush update && rush build",
-    "build-ci": "rush install && rush build",
-    "build-release": "rush install && rush build --release"
-  },
-  "author": "GitHub"
-}

common/config/LICENSE

@@ -1,28 +0,0 @@
-This directory contains content from https://github.com/microsoft/rushstack,
-used under the MIT license as follows.
-See https://github.com/microsoft/rushstack/blob/master/stack/rush-stack/LICENSE.
-
-@microsoft/rush-stack
-
-Copyright (c) Microsoft Corporation. All rights reserved.
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

common/config/rush/.npmrc

@@ -1,12 +0,0 @@
-# Rush uses this file to configure the package registry, regardless of whether the
-# package manager is PNPM, NPM, or Yarn.  Prior to invoking the package manager,
-# Rush will always copy this file to the folder where installation is performed.
-# When NPM is the package manager, Rush works around NPM's processing of
-# undefined environment variables by deleting any lines that reference undefined
-# environment variables.
-#
-# DO NOT SPECIFY AUTHENTICATION CREDENTIALS IN THIS FILE.  It should only be used
-# to configure registry sources.
-
-registry=https://registry.npmjs.org/
-always-auth=false

common/config/rush/command-line.json

@@ -1,32 +0,0 @@
-/**
- * This configuration file defines custom commands for the "rush" command-line.
- * For full documentation, please see https://rushjs.io/pages/configs/command_line_json/
- */
-{
-  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",
-  "commands": [
-    {
-      "commandKind": "bulk",
-      "name": "format",
-      "summary": "Reformat source code in all projects",
-      "description": "Runs the `format` npm task in each project, if present.",
-      "safeForSimultaneousRushProcesses": false,
-      "enableParallelism": true,
-      "ignoreDependencyOrder": true,
-      "ignoreMissingScript": true,
-      "allowWarningsInSuccessfulBuild": false
-    }
-  ],
-  "parameters": [
-    {
-      "parameterKind": "flag",
-      "longName": "--release",
-      "shortName": "-r",
-      "description": "Perform a release build",
-      "associatedCommands": [
-        "build",
-        "rebuild"
-      ],
-    }
-  ]
-}

common/config/rush/common-versions.json

@@ -1,43 +0,0 @@
-/**
- * This configuration file specifies NPM dependency version selections that affect all projects
- * in a Rush repo.  For full documentation, please see https://rushjs.io
- */
-{
-  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/common-versions.schema.json",
-
-  /**
-   * A table that specifies a "preferred version" for a dependency package. The "preferred version"
-   * is typically used to hold an indirect dependency back to a specific version, however generally
-   * it can be any SemVer range specifier (e.g. "~1.2.3"), and it will narrow any (compatible)
-   * SemVer range specifier.  See the Rush documentation for details about this feature.
-   */
-  "preferredVersions": {
-
-    /**
-     * When someone asks for "^1.0.0" make sure they get "1.2.3" when working in this repo,
-     * instead of the latest version.
-     */
-    // "some-library": "1.2.3"
-  },
-
-  /**
-   * The "rush check" command can be used to enforce that every project in the repo must specify
-   * the same SemVer range for a given dependency.  However, sometimes exceptions are needed.
-   * The allowedAlternativeVersions table allows you to list other SemVer ranges that will be
-   * accepted by "rush check" for a given dependency.
-   *
-   * IMPORTANT: THIS TABLE IS FOR *ADDITIONAL* VERSION RANGES THAT ARE ALTERNATIVES TO THE
-   * USUAL VERSION (WHICH IS INFERRED BY LOOKING AT ALL PROJECTS IN THE REPO).
-   * This design avoids unnecessary churn in this file.
-   */
-   "allowedAlternativeVersions": {
-
-    /**
-     * For example, allow some projects to use an older TypeScript compiler
-     * (in addition to whatever "usual" version is being used by other projects in the repo):
-     */
-    // "typescript": [
-    //   "~2.4.0"
-    // ]
-  }
-}

common/config/rush/pnpmfile.js

@@ -1,32 +0,0 @@
-"use strict";
-
-/**
- * When using the PNPM package manager, you can use pnpmfile.js to workaround
- * dependencies that have mistakes in their package.json file.  (This feature is
- * functionally similar to Yarn's "resolutions".)
- *
- * For details, see the PNPM documentation:
- * https://pnpm.js.org/docs/en/hooks.html
- *
- * IMPORTANT: SINCE THIS FILE CONTAINS EXECUTABLE CODE, MODIFYING IT IS LIKELY
- * TO INVALIDATE ANY CACHED DEPENDENCY ANALYSIS.  We recommend to run "rush update --full"
- * after any modification to pnpmfile.js.
- *
- */
-module.exports = {
-  hooks: {
-    readPackage
-  }
-};
-
-/**
- * This hook is invoked during installation before a package's dependencies
- * are selected.
- * The `packageJson` parameter is the deserialized package.json
- * contents for the package that is about to be installed.
- * The `context` parameter provides a log() function.
- * The return value is the updated object.
- */
-function readPackage(packageJson, context) {
-  return packageJson;
-}

common/config/rush/version-policies.json

@@ -1,6 +0,0 @@
-/**
- * This is configuration file is used for advanced publishing configurations with Rush.
- * For full documentation, please see https://rushjs.io/pages/configs/version_policies_json/
- */
-
-[]

common/scripts/install-run-rush.js

@@ -1,52 +0,0 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where the Rush command may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
-// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it.
-// An example usage would be:
-//
-//    node common/scripts/install-run-rush.js install
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const path = require("path");
-const fs = require("fs");
-const install_run_1 = require("./install-run");
-const PACKAGE_NAME = '@microsoft/rush';
-function getRushVersion() {
-    const rushJsonFolder = install_run_1.findRushJsonFolder();
-    const rushJsonPath = path.join(rushJsonFolder, install_run_1.RUSH_JSON_FILENAME);
-    try {
-        const rushJsonContents = fs.readFileSync(rushJsonPath, 'utf-8');
-        // Use a regular expression to parse out the rushVersion value because rush.json supports comments,
-        // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script.
-        const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/);
-        return rushJsonMatches[1];
-    }
-    catch (e) {
-        throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` +
-            'The \'rushVersion\' field is either not assigned in rush.json or was specified ' +
-            'using an unexpected syntax.');
-    }
-}
-function run() {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ ...packageBinArgs /* [build, --to, myproject] */] = process.argv;
-    if (!nodePath || !scriptPath) {
-        throw new Error('Unexpected exception: could not detect node path or script path');
-    }
-    if (process.argv.length < 3) {
-        console.log('Usage: install-run-rush.js <command> [args...]');
-        console.log('Example: install-run-rush.js build --to myproject');
-        process.exit(1);
-    }
-    install_run_1.runWithErrorAndStatusCode(() => {
-        const version = getRushVersion();
-        console.log(`The rush.json configuration requests Rush version ${version}`);
-        return install_run_1.installAndRun(PACKAGE_NAME, version, 'rush', packageBinArgs);
-    });
-}
-run();
-//# sourceMappingURL=install-run-rush.js.map

common/scripts/install-run.js

@@ -1,399 +0,0 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where a Node tool may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the specified
-// version of the specified tool (if not already installed), and then pass a command-line to it.
-// An example usage would be:
-//
-//    node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const childProcess = require("child_process");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-exports.RUSH_JSON_FILENAME = 'rush.json';
-const INSTALLED_FLAG_FILENAME = 'installed.flag';
-const NODE_MODULES_FOLDER_NAME = 'node_modules';
-const PACKAGE_JSON_FILENAME = 'package.json';
-/**
- * Parse a package specifier (in the form of name\@version) into name and version parts.
- */
-function parsePackageSpecifier(rawPackageSpecifier) {
-    rawPackageSpecifier = (rawPackageSpecifier || '').trim();
-    const separatorIndex = rawPackageSpecifier.lastIndexOf('@');
-    let name;
-    let version = undefined;
-    if (separatorIndex === 0) {
-        // The specifier starts with a scope and doesn't have a version specified
-        name = rawPackageSpecifier;
-    }
-    else if (separatorIndex === -1) {
-        // The specifier doesn't have a version
-        name = rawPackageSpecifier;
-    }
-    else {
-        name = rawPackageSpecifier.substring(0, separatorIndex);
-        version = rawPackageSpecifier.substring(separatorIndex + 1);
-    }
-    if (!name) {
-        throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`);
-    }
-    return { name, version };
-}
-/**
- * Resolve a package specifier to a static version
- */
-function resolvePackageVersion(rushCommonFolder, { name, version }) {
-    if (!version) {
-        version = '*'; // If no version is specified, use the latest version
-    }
-    if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) {
-        // If the version contains only characters that we recognize to be used in static version specifiers,
-        // pass the version through
-        return version;
-    }
-    else {
-        // version resolves to
-        try {
-            const rushTempFolder = ensureAndJoinPath(rushCommonFolder, 'temp');
-            const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-            syncNpmrc(sourceNpmrcFolder, rushTempFolder);
-            const npmPath = getNpmPath();
-            // This returns something that looks like:
-            //  @microsoft/rush@3.0.0 '3.0.0'
-            //  @microsoft/rush@3.0.1 '3.0.1'
-            //  ...
-            //  @microsoft/rush@3.0.20 '3.0.20'
-            //  <blank line>
-            const npmVersionSpawnResult = childProcess.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], {
-                cwd: rushTempFolder,
-                stdio: []
-            });
-            if (npmVersionSpawnResult.status !== 0) {
-                throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`);
-            }
-            const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString();
-            const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line);
-            const latestVersion = versionLines[versionLines.length - 1];
-            if (!latestVersion) {
-                throw new Error('No versions found for the specified version range.');
-            }
-            const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/);
-            if (!versionMatches) {
-                throw new Error(`Invalid npm output ${latestVersion}`);
-            }
-            return versionMatches[1];
-        }
-        catch (e) {
-            throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`);
-        }
-    }
-}
-let _npmPath = undefined;
-/**
- * Get the absolute path to the npm executable
- */
-function getNpmPath() {
-    if (!_npmPath) {
-        try {
-            if (os.platform() === 'win32') {
-                // We're on Windows
-                const whereOutput = childProcess.execSync('where npm', { stdio: [] }).toString();
-                const lines = whereOutput.split(os.EOL).filter((line) => !!line);
-                // take the last result, we are looking for a .cmd command
-                // see https://github.com/Microsoft/web-build-tools/issues/759
-                _npmPath = lines[lines.length - 1];
-            }
-            else {
-                // We aren't on Windows - assume we're on *NIX or Darwin
-                _npmPath = childProcess.execSync('which npm', { stdio: [] }).toString();
-            }
-        }
-        catch (e) {
-            throw new Error(`Unable to determine the path to the NPM tool: ${e}`);
-        }
-        _npmPath = _npmPath.trim();
-        if (!fs.existsSync(_npmPath)) {
-            throw new Error('The NPM executable does not exist');
-        }
-    }
-    return _npmPath;
-}
-exports.getNpmPath = getNpmPath;
-let _rushJsonFolder;
-/**
- * Find the absolute path to the folder containing rush.json
- */
-function findRushJsonFolder() {
-    if (!_rushJsonFolder) {
-        let basePath = __dirname;
-        let tempPath = __dirname;
-        do {
-            const testRushJsonPath = path.join(basePath, exports.RUSH_JSON_FILENAME);
-            if (fs.existsSync(testRushJsonPath)) {
-                _rushJsonFolder = basePath;
-                break;
-            }
-            else {
-                basePath = tempPath;
-            }
-        } while (basePath !== (tempPath = path.dirname(basePath))); // Exit the loop when we hit the disk root
-        if (!_rushJsonFolder) {
-            throw new Error('Unable to find rush.json.');
-        }
-    }
-    return _rushJsonFolder;
-}
-exports.findRushJsonFolder = findRushJsonFolder;
-/**
- * Create missing directories under the specified base directory, and return the resolved directory.
- *
- * Does not support "." or ".." path segments.
- * Assumes the baseFolder exists.
- */
-function ensureAndJoinPath(baseFolder, ...pathSegments) {
-    let joinedPath = baseFolder;
-    try {
-        for (let pathSegment of pathSegments) {
-            pathSegment = pathSegment.replace(/[\\\/]/g, '+');
-            joinedPath = path.join(joinedPath, pathSegment);
-            if (!fs.existsSync(joinedPath)) {
-                fs.mkdirSync(joinedPath);
-            }
-        }
-    }
-    catch (e) {
-        throw new Error(`Error building local installation folder (${path.join(baseFolder, ...pathSegments)}): ${e}`);
-    }
-    return joinedPath;
-}
-/**
- * As a workaround, _syncNpmrc() copies the .npmrc file to the target folder, and also trims
- * unusable lines from the .npmrc file.  If the source .npmrc file not exist, then _syncNpmrc()
- * will delete an .npmrc that is found in the target folder.
- *
- * Why are we trimming the .npmrc lines?  NPM allows environment variables to be specified in
- * the .npmrc file to provide different authentication tokens for different registry.
- * However, if the environment variable is undefined, it expands to an empty string, which
- * produces a valid-looking mapping with an invalid URL that causes an error.  Instead,
- * we'd prefer to skip that line and continue looking in other places such as the user's
- * home directory.
- *
- * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc()
- */
-function syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder) {
-    const sourceNpmrcPath = path.join(sourceNpmrcFolder, '.npmrc');
-    const targetNpmrcPath = path.join(targetNpmrcFolder, '.npmrc');
-    try {
-        if (fs.existsSync(sourceNpmrcPath)) {
-            let npmrcFileLines = fs.readFileSync(sourceNpmrcPath).toString().split('\n');
-            npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim());
-            const resultLines = [];
-            // Trim out lines that reference environment variables that aren't defined
-            for (const line of npmrcFileLines) {
-                // This finds environment variable tokens that look like "${VAR_NAME}"
-                const regex = /\$\{([^\}]+)\}/g;
-                const environmentVariables = line.match(regex);
-                let lineShouldBeTrimmed = false;
-                if (environmentVariables) {
-                    for (const token of environmentVariables) {
-                        // Remove the leading "${" and the trailing "}" from the token
-                        const environmentVariableName = token.substring(2, token.length - 1);
-                        if (!process.env[environmentVariableName]) {
-                            lineShouldBeTrimmed = true;
-                            break;
-                        }
-                    }
-                }
-                if (lineShouldBeTrimmed) {
-                    // Example output:
-                    // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}"
-                    resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line);
-                }
-                else {
-                    resultLines.push(line);
-                }
-            }
-            fs.writeFileSync(targetNpmrcPath, resultLines.join(os.EOL));
-        }
-        else if (fs.existsSync(targetNpmrcPath)) {
-            // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target
-            fs.unlinkSync(targetNpmrcPath);
-        }
-    }
-    catch (e) {
-        throw new Error(`Error syncing .npmrc file: ${e}`);
-    }
-}
-/**
- * Detects if the package in the specified directory is installed
- */
-function isPackageAlreadyInstalled(packageInstallFolder) {
-    try {
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        if (!fs.existsSync(flagFilePath)) {
-            return false;
-        }
-        const fileContents = fs.readFileSync(flagFilePath).toString();
-        return fileContents.trim() === process.version;
-    }
-    catch (e) {
-        return false;
-    }
-}
-/**
- * Removes the following files and directories under the specified folder path:
- *  - installed.flag
- *  -
- *  - node_modules
- */
-function cleanInstallFolder(rushCommonFolder, packageInstallFolder) {
-    try {
-        const flagFile = path.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        if (fs.existsSync(flagFile)) {
-            fs.unlinkSync(flagFile);
-        }
-        const packageLockFile = path.resolve(packageInstallFolder, 'package-lock.json');
-        if (fs.existsSync(packageLockFile)) {
-            fs.unlinkSync(packageLockFile);
-        }
-        const nodeModulesFolder = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME);
-        if (fs.existsSync(nodeModulesFolder)) {
-            const rushRecyclerFolder = ensureAndJoinPath(rushCommonFolder, 'temp', 'rush-recycler', `install-run-${Date.now().toString()}`);
-            fs.renameSync(nodeModulesFolder, rushRecyclerFolder);
-        }
-    }
-    catch (e) {
-        throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`);
-    }
-}
-function createPackageJson(packageInstallFolder, name, version) {
-    try {
-        const packageJsonContents = {
-            'name': 'ci-rush',
-            'version': '0.0.0',
-            'dependencies': {
-                [name]: version
-            },
-            'description': 'DON\'T WARN',
-            'repository': 'DON\'T WARN',
-            'license': 'MIT'
-        };
-        const packageJsonPath = path.join(packageInstallFolder, PACKAGE_JSON_FILENAME);
-        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2));
-    }
-    catch (e) {
-        throw new Error(`Unable to create package.json: ${e}`);
-    }
-}
-/**
- * Run "npm install" in the package install folder.
- */
-function installPackage(packageInstallFolder, name, version) {
-    try {
-        console.log(`Installing ${name}...`);
-        const npmPath = getNpmPath();
-        const result = childProcess.spawnSync(npmPath, ['install'], {
-            stdio: 'inherit',
-            cwd: packageInstallFolder,
-            env: process.env
-        });
-        if (result.status !== 0) {
-            throw new Error('"npm install" encountered an error');
-        }
-        console.log(`Successfully installed ${name}@${version}`);
-    }
-    catch (e) {
-        throw new Error(`Unable to install package: ${e}`);
-    }
-}
-/**
- * Get the ".bin" path for the package.
- */
-function getBinPath(packageInstallFolder, binName) {
-    const binFolderPath = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin');
-    const resolvedBinName = (os.platform() === 'win32') ? `${binName}.cmd` : binName;
-    return path.resolve(binFolderPath, resolvedBinName);
-}
-/**
- * Write a flag file to the package's install directory, signifying that the install was successful.
- */
-function writeFlagFile(packageInstallFolder) {
-    try {
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        fs.writeFileSync(flagFilePath, process.version);
-    }
-    catch (e) {
-        throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`);
-    }
-}
-function installAndRun(packageName, packageVersion, packageBinName, packageBinArgs) {
-    const rushJsonFolder = findRushJsonFolder();
-    const rushCommonFolder = path.join(rushJsonFolder, 'common');
-    const packageInstallFolder = ensureAndJoinPath(rushCommonFolder, 'temp', 'install-run', `${packageName}@${packageVersion}`);
-    if (!isPackageAlreadyInstalled(packageInstallFolder)) {
-        // The package isn't already installed
-        cleanInstallFolder(rushCommonFolder, packageInstallFolder);
-        const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-        syncNpmrc(sourceNpmrcFolder, packageInstallFolder);
-        createPackageJson(packageInstallFolder, packageName, packageVersion);
-        installPackage(packageInstallFolder, packageName, packageVersion);
-        writeFlagFile(packageInstallFolder);
-    }
-    const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`;
-    const statusMessageLine = new Array(statusMessage.length + 1).join('-');
-    console.log(os.EOL + statusMessage + os.EOL + statusMessageLine + os.EOL);
-    const binPath = getBinPath(packageInstallFolder, packageBinName);
-    const result = childProcess.spawnSync(binPath, packageBinArgs, {
-        stdio: 'inherit',
-        cwd: process.cwd(),
-        env: process.env
-    });
-    return result.status;
-}
-exports.installAndRun = installAndRun;
-function runWithErrorAndStatusCode(fn) {
-    process.exitCode = 1;
-    try {
-        const exitCode = fn();
-        process.exitCode = exitCode;
-    }
-    catch (e) {
-        console.error(os.EOL + os.EOL + e.toString() + os.EOL + os.EOL);
-    }
-}
-exports.runWithErrorAndStatusCode = runWithErrorAndStatusCode;
-function run() {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ rawPackageSpecifier, /* qrcode@^1.2.0 */ packageBinName, /* qrcode */ ...packageBinArgs /* [-f, myproject/lib] */] = process.argv;
-    if (!nodePath) {
-        throw new Error('Unexpected exception: could not detect node path');
-    }
-    if (path.basename(scriptPath).toLowerCase() !== 'install-run.js') {
-        // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control
-        // to the script that (presumably) imported this file
-        return;
-    }
-    if (process.argv.length < 4) {
-        console.log('Usage: install-run.js <package>@<version> <command> [args...]');
-        console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io');
-        process.exit(1);
-    }
-    runWithErrorAndStatusCode(() => {
-        const rushJsonFolder = findRushJsonFolder();
-        const rushCommonFolder = ensureAndJoinPath(rushJsonFolder, 'common');
-        const packageSpecifier = parsePackageSpecifier(rawPackageSpecifier);
-        const name = packageSpecifier.name;
-        const version = resolvePackageVersion(rushCommonFolder, packageSpecifier);
-        if (packageSpecifier.version !== version) {
-            console.log(`Resolved to ${name}@${version}`);
-        }
-        return installAndRun(name, version, packageBinName, packageBinArgs);
-    });
-}
-run();
-//# sourceMappingURL=install-run.js.map

configs/typescript-config/extension.tsconfig.json

@@ -1,8 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/tsconfig",
-  "extends": "./common.tsconfig.json",
-  "compilerOptions": {
-    "declaration": false,
-    "strict": true
-  }
-}

configs/typescript-config/lib.tsconfig.json

@@ -1,4 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/tsconfig",
-  "extends": "./common.tsconfig.json"
-}

configs/typescript-config/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "typescript-config",
-  "description": "TypeScript configurations",
-  "author": "GitHub",
-  "private": true,
-  "version": "0.0.1",
-  "publisher": "GitHub",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/github/vscode-codeql"
-  },
-  "scripts": {
-    "build": "",
-    "format": ""
-  },
-  "devDependencies": {},
-  "dependencies": {}
-}

extensions/ql-vscode/.eslintrc.js

@@ -0,0 +1,35 @@
+module.exports = {
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    ecmaVersion: 2018,
+    sourceType: "module",
+    project: ["tsconfig.json", "./src/**/tsconfig.json", "./gulpfile.ts/tsconfig.json"],
+  },
+  plugins: ["@typescript-eslint"],
+  env: {
+    node: true,
+    es6: true,
+  },
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  rules: {
+    "@typescript-eslint/no-use-before-define": 0,
+    "@typescript-eslint/no-unused-vars": [
+      "warn",
+      {
+        vars: "all",
+        args: "none",
+        ignoreRestSiblings: false,
+      },
+    ],
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/no-non-null-assertion": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "prefer-const": ["warn", { destructuring: "all" }],
+    indent: "off",
+    "@typescript-eslint/indent": "off",
+    "@typescript-eslint/no-throw-literal": "error",
+    "no-useless-escape": 0,
+    semi: 2,
+    quotes: ["warn", "single"]
+  },
+};

extensions/ql-vscode/CHANGELOG.md

@@ -1,13 +1,168 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.3.6 - 4 November 2020
+
+- Fix URI encoding for databases that were created with special characters in their paths. [#648](https://github.com/github/vscode-codeql/pull/648)
+- Disable CodeQL Test commands from the command palette [#667](https://github.com/github/vscode-codeql/pull/667)
+- Fix display of booleans in results view. [#657](https://github.com/github/vscode-codeql/pull/657)
+- Avoid recursive selection changes in AST Viewer. [#668](https://github.com/github/vscode-codeql/pull/668)
+
+## 1.3.5 - 27 October 2020
+
+- Fix a bug where archived source folders for databases were not showing any contents.
+
+## 1.3.4 - 22 October 2020
+
+- Add friendly welcome message when the databases view is empty.
+- Add open query, open results, and remove query commands in the query history view title bar.
+- The maximum number of simultaneous queries launchable by the `CodeQL: Run Queries in Selected Files` command is now configurable by changing the `codeQL.runningQueries.maxQueries` setting.
+- Allow simultaneously run queries to be canceled in a single-click.
+- Prevent multiple upgrade dialogs from appearing when running simultaneous queries on upgradeable databases.
+- Fix sorting of results. Some pages of results would have the wrong sort order and columns.
+- Remember previous sort order when reloading query results.
+- Fix proper escaping of backslashes in SARIF message strings.
+- Allow setting `codeQL.runningQueries.numberOfThreads` and `codeQL.runningTests.numberOfThreads` to 0, (which is interpreted as 'use one thread per core on the machine').
+- Clear the problems view of all CodeQL query results when a database is removed.
+- Add a `View DIL` command on query history items. This opens a text editor containing the Datalog Intermediary Language representation of the compiled query.
+- Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/exploring-the-structure-of-your-source-code.html).
+- The `codeQL.runningTests.numberOfThreads` setting is now used correctly when running tests.
+
+## 1.3.3 - 16 September 2020
+
+- Fix display of raw results entities with label but no url.
+- Fix bug where sort order is forgotten when changing raw results page.
+- Avoid showing a location link in results view when a result item has an empty location.
+
+## 1.3.2 - 12 August 2020
+
+- Fix error with choosing qlpack search path.
+- Fix pagination when there are no results.
+- Suppress database downloaded from URL message when action canceled.
+- Fix QL test discovery to avoid showing duplicate tests in the test explorer.
+- Enable pagination of query results
+- Add experimental AST Viewer for Go and C++. To enable, add `"codeQL.experimentalAstViewer": true` to the user settings file.
+
+## 1.3.1 - 7 July 2020
+
+- Fix unzipping of large files.
+- Ensure compare order is consistent when selecting two queries to compare. The first query selected is always the _from_ query and the query selected later is always the _to_ query.
+- Ensure added databases have zipped source locations for databases added as archives or downloaded from the internet.
+- Fix bug where it is not possible to add databases starting with `db-*`.
+- Change styling of pagination section of the results page.
+- Fix display of query text for stored quick queries.
+
+## 1.3.0 - 22 June 2020
+
+- Report error when selecting invalid database.
+- Add descriptive message for database archive import failure.
+- Respect VS Code's i18n locale setting when formatting dates and sorting strings.
+- Allow the opening of large SARIF files externally from VS Code.
+- Add new 'CodeQL: Compare Query' command that shows the differences between two queries.
+- Allow multiple items in the query history view to be removed in one operation.
+- Allow multiple items in the databases view to be removed in one operation.
+- Allow multiple items in the databases view to be upgraded in one operation.
+- Allow multiple items in the databases view to have their external folders opened.
+- Allow all selected queries to be run in one command from the file explorer.
+
+## 1.2.2 - 8 June 2020
+
+- Fix auto-indentation rules.
+- Add ability to download platform-specific releases of the CodeQL CLI if they are available.
+- Fix handling of downloading prerelease versions of the CodeQL CLI.
+- Add pagination for displaying non-interpreted results.
+
+## 1.2.1 - 29 May 2020
+
+- Better formatting and autoindentation when adding QLDoc comments to `.ql` and `.qll` files.
+- Allow for more flexibility when opening a database in the workspace. A user can now choose the actual database folder, or the nested `db-*` folder.
+- Add query history menu command for viewing corresponding SARIF file.
+- Add ability for users to download databases directly from LGTM.com.
+
+## 1.2.0 - 19 May 2020
+
+- Enable 'Go to Definition' and 'Go to References' on source archive
+  files in CodeQL databases. This is handled by a CodeQL query.
+- Fix adding database archive files on Windows.
+- Enable adding remote and local database archive files from the
+  command palette.
+
+## 1.1.5 - 15 May 2020
+
+- Links in results are no longer underlined and monospaced.
+- Add the ability to choose a database either from an archive, a folder, or from the internet.
+- New icons for commands on the databases view.
+
+## 1.1.4 - 13 May 2020
+
+- Add the ability to download and install databases archives from the internet.
+
+## 1.1.3 - 8 May 2020
+
+- Add a suggestion in alerts view to view raw results, when there are
+  raw results but no alerts.
+- Add the ability to rename databases in the database view.
+- Add the ability to open the directory in the filesystem
+  of a database.
+
+## 1.1.2 - 28 April 2020
+
+- Implement syntax highlighting for the new `unique` aggregate.
+- Implement XML syntax highlighting for `.qhelp` files.
+- Add option to auto save queries before running them.
+- Add new command in query history to view the query text of the
+  selected query (note that this may be different from the current
+  contents of the query file if the file has been edited).
+- Add ability to sort CodeQL databases by name or by date added.
+
+## 1.1.1 - 23 March 2020
+
+- Fix quick evaluation in `.qll` files.
+- Add new command in query history view to view the log file of a
+  query.
+- Request user acknowledgment before updating the CodeQL binaries.
+- Warn when using the deprecated `codeql.cmd` launcher on Windows.
+
+## 1.1.0 - 17 March 2020
+
+- Add functionality for testing custom CodeQL queries by using the VS
+  Code Test Explorer extension and `codeql test`. See the documentation for
+  more details.
+- Add a "Show log" button to all information, error, and warning
+  popups that will display the CodeQL extension log.
+- Display a message when a query times out.
+- Show canceled queries in query history.
+- Improve error messages when attempting to run non-query files.
+
+## 1.0.6 - 28 February 2020
+
+- Add command to restart query server.
+- Enable support for future minor upgrades to the CodeQL CLI.
+
+## 1.0.5 - 13 February 2020
+
+- Add an icon next to any failed query runs in the query history
+  view.
+- Add the ability to sort alerts by alert message.
+
+## 1.0.4 - 24 January 2020
+
+- Disable word-based autocomplete by default.
+- Add command `CodeQL: Quick Query` for easy query creation without
+  having to choose a place in the filesystem to store the query file.
+
+## 1.0.3 - 13 January 2020
+
+- Reduce the frequency of CodeQL CLI update checks to help avoid hitting GitHub API limits of 60 requests per
+  hour for unauthenticated IPs.
+- Fix sorting of result sets with names containing special characters.
+
 ## 1.0.2 - 13 December 2019
 
 - Fix rendering of negative numbers in results.
 - Allow customization of query history labels from settings and from
   query history view context menu.
 - Show number of results in results view.
-- Add commands `CodeQL: Show Next Step on Path` and `CodeQL: Show
-  Previous Step on Path` for navigating the steps on the currently
+- Add commands `CodeQL: Show Next Step on Path` and `CodeQL: Show Previous Step on Path` for navigating the steps on the currently
   shown path result.
 
 ## 1.0.1 - 21 November 2019
@@ -18,7 +173,6 @@
 - Fix the automatic upgrading of CodeQL databases when using upgrade scripts from the workspace.
 - Allow removal of items from the CodeQL Query History view.
 
-
 ## 1.0.0 - 14 November 2019
 
 Initial release of CodeQL for Visual Studio Code.

extensions/ql-vscode/README.md

@@ -2,30 +2,30 @@
 
 This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
 
-* Enables you to use CodeQL to query databases generated from source code.
-* Shows the flow of data through the results of path queries, which is essential for triaging security results.
-* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/Semmle/ql).
-* Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
+- Enables you to use CodeQL to query databases generated from source code.
+- Shows the flow of data through the results of path queries, which is essential for triaging security results.
+- Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
+- Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
-To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
+To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
 For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
 
-**Quick start: Installing and configuring the extension**
+### Quick start: Installing and configuring the extension
 
 1. [Install the extension](#installing-the-extension).
 1. [Check access to the CodeQL CLI](#checking-access-to-the-codeql-cli).
 1. [Clone the CodeQL starter workspace](#cloning-the-codeql-starter-workspace).
 
-**Quick start: Using CodeQL**
+### Quick start: Using CodeQL
 
 1. [Import a database from LGTM](#importing-a-database-from-lgtm).
 1. [Run a query](#running-a-query).
 
------
+---
 
 ## Quick start: Installing and configuring the extension
 
@@ -49,11 +49,26 @@ If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Lo
 ### Cloning the CodeQL starter workspace
 
 When you're working with CodeQL, you need access to the standard CodeQL libraries and queries.
-Initially, we recommend that you clone and use the ready-to-use starter workspace, https://github.com/github/vscode-codeql-starter/.
+Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
 For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
 
+## Upgrading CodeQL standard libraries
+
+You can easily keep up-to-date with the latest changes to the [CodeQL standard libraries](https://github.com/github/codeql).
+
+If you're using the [CodeQL starter workspace](https://github.com/github/vscode-codeql-starter/), you can pull in the latest standard libraries by running:
+
+```shell
+git pull
+git submodule update --recursive
+```
+
+in the starter workspace directory.
+
+If you're using your own clone of the CodeQL standard libraries, you can do a `git pull` from where you have the libraries checked out.
+
 ## Quick start: Using CodeQL
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.
@@ -62,16 +77,13 @@ You can find all the commands contributed by the extension in the Command Palett
 
 While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
 
-1. Log in to LGTM.com.
-1. Find a project you're interested in and display the **Integrations** tab (for example, [Apache Kafka](https://lgtm.com/projects/g/apache/kafka/ci/)).
-1. Scroll to the **CodeQL databases for local analysis** section at the bottom of the page.
-1. Download databases for the languages that you want to explore.
-1. Unzip the databases.
-1. For each database that you want to import:
-    1. In the VS Code sidebar, go to **CodeQL** > **Databases** and click **+**.
-    1. Browse to the unzipped database folder (the parent folder that contains `db-<language>` and `src`) and select **Choose database** to add it.
-
-When the import is complete, each CodeQL database is displayed in the CodeQL sidebar under **Databases**.
+1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
+1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
+1. Copy the link to that project, for example `https://lgtm.com/projects/g/apache/kafka`.
+1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from LGTM** command.
+1. Paste the link you copied earlier.
+1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
+1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
 ### Running a query
 
@@ -79,7 +91,7 @@ The instructions below assume that you're using the CodeQL starter workspace, or
 
 1. Expand the `ql` folder and locate a query to run. The standard queries are grouped by target language and then type, for example: `ql/java/ql/src/Likely Bugs`.
 1. Open a query (`.ql`) file.
-3. Right-click in the query window and select **CodeQL: Run Query**. Alternatively, open the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**), type `Run Query`, then select **CodeQL: Run Query**.
+1. Right-click in the query window and select **CodeQL: Run Query**. Alternatively, open the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**), type `Run Query`, then select **CodeQL: Run Query**.
 
 The CodeQL extension runs the query on the current database using the CLI and reports progress in the bottom right corner of the application.
 When the results are ready, they're displayed in the CodeQL Query Results view. Use the dropdown menu to choose between different forms of result output.
@@ -90,10 +102,10 @@ If there are any problems running a query, a notification is displayed in the bo
 
 For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
 
-* [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
-* [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-* [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
-* [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
+- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
+- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
 

extensions/ql-vscode/gulpfile.js/index.js

@@ -1,19 +0,0 @@
-'use strict';
-
-require('ts-node').register({});
-const gulp = require('gulp');
-const {
-  compileTypeScript,
-  watchTypeScript,
-  packageExtension,
-  compileTextMateGrammar,
-  copyTestData,
-  copyViewCss
-} = require('build-tasks');
-const { compileView } = require('./webpack');
-
-exports.buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
-exports.compileTextMateGrammar = compileTextMateGrammar;
-exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);
-exports.watchTypeScript = watchTypeScript;
-exports.compileTypeScript = compileTypeScript;

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -0,0 +1,72 @@
+import * as fs from 'fs-extra';
+import * as jsonc from 'jsonc-parser';
+import * as path from 'path';
+
+export interface DeployedPackage {
+  distPath: string;
+  name: string;
+  version: string;
+}
+
+const packageFiles = [
+  '.vscodeignore',
+  'CHANGELOG.md',
+  'README.md',
+  'language-configuration.json',
+  'media',
+  'node_modules',
+  'out'
+];
+
+async function copyPackage(sourcePath: string, destPath: string): Promise<void> {
+  for (const file of packageFiles) {
+    console.log(`copying ${path.resolve(sourcePath, file)} to ${path.resolve(destPath, file)}`);
+    await fs.copy(path.resolve(sourcePath, file), path.resolve(destPath, file));
+  }
+}
+
+export async function deployPackage(packageJsonPath: string): Promise<DeployedPackage> {
+  try {
+    const packageJson: any = jsonc.parse(await fs.readFile(packageJsonPath, 'utf8'));
+
+    // Default to development build; use flag --release to indicate release build.
+    const isDevBuild = !process.argv.includes('--release');
+    const distDir = path.join(__dirname, '../../../dist');
+    await fs.mkdirs(distDir);
+
+    if (isDevBuild) {
+      // NOTE: rootPackage.name had better not have any regex metacharacters
+      const oldDevBuildPattern = new RegExp('^' + packageJson.name + '[^/]+-dev[0-9.]+\\.vsix$');
+      // Dev package filenames are of the form
+      //    vscode-codeql-0.0.1-dev.2019.9.27.19.55.20.vsix
+      (await fs.readdir(distDir)).filter(name => name.match(oldDevBuildPattern)).map(build => {
+        console.log(`Deleting old dev build ${build}...`);
+        fs.unlinkSync(path.join(distDir, build));
+      });
+      const now = new Date();
+      packageJson.version = packageJson.version +
+        `-dev.${now.getUTCFullYear()}.${now.getUTCMonth() + 1}.${now.getUTCDate()}` +
+        `.${now.getUTCHours()}.${now.getUTCMinutes()}.${now.getUTCSeconds()}`;
+    }
+
+    const distPath = path.join(distDir, packageJson.name);
+    await fs.remove(distPath);
+    await fs.mkdirs(distPath);
+
+    await fs.writeFile(path.join(distPath, 'package.json'), JSON.stringify(packageJson, null, 2));
+
+    const sourcePath = path.join(__dirname, '..');
+    console.log(`Copying package '${packageJson.name}' and its dependencies to '${distPath}'...`);
+    await copyPackage(sourcePath, distPath);
+
+    return {
+      distPath: distPath,
+      name: packageJson.name,
+      version: packageJson.version
+    };
+  }
+  catch (e) {
+    console.error(e);
+    throw e;
+  }
+}

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -0,0 +1,10 @@
+import * as gulp from 'gulp';
+import { compileTypeScript, watchTypeScript, copyViewCss } from './typescript';
+import { compileTextMateGrammar } from './textmate';
+import { copyTestData } from './tests';
+import { compileView } from './webpack';
+import { packageExtension } from './package';
+
+export const buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
+export { compileTextMateGrammar, watchTypeScript, compileTypeScript };
+exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);

extensions/ql-vscode/gulpfile.ts/package.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import { deployPackage } from './deploy';
-import * as child_process from 'child-process-promise';
+import * as childProcess from 'child-process-promise';
 
 export async function packageExtension(): Promise<void> {
   const deployedPackage = await deployPackage(path.resolve('package.json'));
@@ -9,7 +9,7 @@ export async function packageExtension(): Promise<void> {
     'package',
     '--out', path.resolve(deployedPackage.distPath, '..', `${deployedPackage.name}-${deployedPackage.version}.vsix`)
   ];
-  const proc = child_process.spawn('vsce', args, {
+  const proc = childProcess.spawn('./node_modules/.bin/vsce', args, {
     cwd: deployedPackage.distPath
   });
   proc.childProcess.stdout!.on('data', (data) => {

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -1,5 +1,5 @@
 import * as gulp from 'gulp';
-import * as js_yaml from 'js-yaml';
+import * as jsYaml from 'js-yaml';
 import * as through from 'through2';
 import * as PluginError from 'plugin-error';
 import * as Vinyl from 'vinyl';
@@ -13,9 +13,10 @@ import * as Vinyl from 'vinyl';
  */
 function replaceReferencesWithStrings(value: string, replacements: Map<string, string>): string {
   let result = value;
+  // eslint-disable-next-line no-constant-condition
   while (true) {
     const original = result;
-    for (const key of replacements.keys()) {
+    for (const key of Array.from(replacements.keys())) {
       result = result.replace(`(?#${key})`, `(?:${replacements.get(key)})`);
     }
     if (result === original) {
@@ -32,7 +33,7 @@ function replaceReferencesWithStrings(value: string, replacements: Map<string, s
  */
 function gatherMacros(yaml: any): Map<string, string> {
   const macros = new Map<string, string>();
-  for (var key in yaml.macros) {
+  for (const key in yaml.macros) {
     macros.set(key, yaml.macros[key]);
   }
 
@@ -55,7 +56,7 @@ function getNodeMatchText(rule: any): string {
   else if (rule.patterns !== undefined) {
     const patterns: string[] = [];
     // For a list of patterns, use the disjunction of those patterns.
-    for (var patternIndex in rule.patterns) {
+    for (const patternIndex in rule.patterns) {
       const pattern = rule.patterns[patternIndex];
       if (pattern.include !== null) {
         patterns.push('(?' + pattern.include + ')');
@@ -65,7 +66,7 @@ function getNodeMatchText(rule: any): string {
     return '(?:' + patterns.join('|') + ')';
   }
   else {
-    return ''
+    return '';
   }
 }
 
@@ -78,7 +79,7 @@ function getNodeMatchText(rule: any): string {
  */
 function gatherMatchTextForRules(yaml: any): Map<string, string> {
   const replacements = new Map<string, string>();
-  for (var key in yaml.repository) {
+  for (const key in yaml.repository) {
     const node = yaml.repository[key];
     replacements.set(key, getNodeMatchText(node));
   }
@@ -106,7 +107,7 @@ function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
  * @param action Callback to invoke on each rule.
  */
 function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
-  for (var key in ruleMap) {
+  for (const key in ruleMap) {
     const rule = ruleMap[key];
     if ((typeof rule) === 'object') {
       action(rule);
@@ -124,7 +125,7 @@ function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
  * @param action The transformation to make on each match pattern.
  */
 function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
-  for (var key in rule) {
+  for (const key in rule) {
     switch (key) {
       case 'begin':
       case 'end':
@@ -184,10 +185,10 @@ function transformFile(yaml: any) {
   visitAllRulesInFile(yaml, (rule) => {
     visitAllMatchesInRule(rule, (match) => {
       if ((typeof match) === 'object') {
-        for (var key in match) {
+        for (const key in match) {
           return macros.get(key)!.replace('(?#)', `(?:${match[key]})`);
         }
-        throw new Error("No key in macro map.")
+        throw new Error('No key in macro map.');
       }
       else {
         return match;
@@ -225,7 +226,7 @@ export function transpileTextMateGrammar() {
     else if (file.isBuffer()) {
       const buf: Buffer = file.contents;
       const yamlText: string = buf.toString('utf8');
-      const jsonData: any = js_yaml.safeLoad(yamlText);
+      const jsonData: any = jsYaml.safeLoad(yamlText);
       transformFile(jsonData);
 
       file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), 'utf8');

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -1,15 +1,14 @@
 {
+  "$schema": "http://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
+    "strict": true,
     "module": "commonjs",
     "target": "es2017",
-    "outDir": "out",
-    "lib": [
-      "es6"
-    ],
+    "lib": ["es6"],
     "moduleResolution": "node",
     "sourceMap": true,
-    "rootDir": "../../src",
+    "rootDir": ".",
     "strictNullChecks": true,
     "noFallthroughCasesInSwitch": true,
     "preserveWatchOutput": true,
@@ -19,12 +18,5 @@
     "noUnusedLocals": true,
     "noUnusedParameters": true
   },
-  "include": [
-    "../../src/**/*.ts"
-  ],
-  "exclude": [
-    "../../node_modules",
-    "../../test",
-    "../../**/view"
-  ]
-}
+  "include": ["*.ts"]
+}

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -0,0 +1,42 @@
+import * as colors from 'ansi-colors';
+import * as gulp from 'gulp';
+import * as sourcemaps from 'gulp-sourcemaps';
+import * as ts from 'gulp-typescript';
+
+function goodReporter(): ts.reporter.Reporter {
+  return {
+    error: (error, typescript) => {
+      if (error.tsFile) {
+        console.log('[' + colors.gray('gulp-typescript') + '] ' + colors.red(error.fullFilename
+          + '(' + (error.startPosition!.line + 1) + ',' + error.startPosition!.character + '): ')
+          + 'error TS' + error.diagnostic.code + ': ' + typescript.flattenDiagnosticMessageText(error.diagnostic.messageText, '\n'));
+      }
+      else {
+        console.log(error.message);
+      }
+    },
+  };
+}
+
+const tsProject = ts.createProject('tsconfig.json');
+
+export function compileTypeScript() {
+  return tsProject.src()
+    .pipe(sourcemaps.init())
+    .pipe(tsProject(goodReporter()))
+    .pipe(sourcemaps.write('.', {
+      includeContent: false,
+      sourceRoot: '.',
+    }))
+    .pipe(gulp.dest('out'));
+}
+
+export function watchTypeScript() {
+  gulp.watch('src/**/*.ts', compileTypeScript);
+}
+
+/** Copy CSS files for the results view into the output directory. */
+export function copyViewCss() {
+  return gulp.src('src/view/*.css')
+    .pipe(gulp.dest('out'));
+}

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -4,13 +4,14 @@ import * as webpack from 'webpack';
 export const config: webpack.Configuration = {
   mode: 'development',
   entry: {
-    resultsView: './src/view/results.tsx'
+    resultsView: './src/view/results.tsx',
+    compareView: './src/compare/view/Compare.tsx',
   },
   output: {
     path: path.resolve(__dirname, '..', 'out'),
-    filename: "[name].js"
+    filename: '[name].js'
   },
-  devtool: 'source-map',
+  devtool: 'inline-source-map',
   resolve: {
     extensions: ['.js', '.ts', '.tsx', '.json']
   },
@@ -19,6 +20,9 @@ export const config: webpack.Configuration = {
       {
         test: /\.(ts|tsx)$/,
         loader: 'ts-loader',
+        options: {
+          configFile: 'src/view/tsconfig.json',
+        }
       },
       {
         test: /\.less$/,

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -15,7 +15,8 @@ export function compileView(cb: (err?: Error) => void) {
       hash: false,
       entrypoints: false,
       timings: false,
-      modules: false
+      modules: false,
+      errors: true
     }));
     if (stats.hasErrors()) {
       cb(new Error('Compilation errors detected.'));

extensions/ql-vscode/language-configuration.json

@@ -1,72 +1,34 @@
 {
-    "comments": {
-        // symbol used for single line comment. Remove this entry if your language does not support line comments
-        "lineComment": "//",
-        // symbols used for start and end a block comment. Remove this entry if your language does not support block comments
-        "blockComment": [
-            "/*",
-            "*/"
-        ]
-    },
-    // symbols used as brackets
-    "brackets": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ]
-    ],
-    // symbols that are auto closed when typing
-    "autoClosingPairs": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ],
-        [
-            "\"",
-            "\""
-        ],
-        [
-            "'",
-            "'"
-        ]
-    ],
-    // symbols that that can be used to surround a selection
-    "surroundingPairs": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ],
-        [
-            "\"",
-            "\""
-        ],
-        [
-            "'",
-            "'"
-        ]
-    ]
-}
+  "comments": {
+    "lineComment": "//",
+    "blockComment": ["/*", "*/"]
+  },
+  "brackets": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"]
+  ],
+  "autoClosingPairs": [
+    { "open": "{", "close": "}" },
+    { "open": "[", "close": "]" },
+    { "open": "(", "close": ")" },
+    { "open": "'", "close": "'", "notIn": ["string", "comment"] },
+    { "open": "\"", "close": "\"", "notIn": ["string"] },
+    { "open": "/**", "close": " */", "notIn": ["string"] }
+  ],
+  "autoCloseBefore": ";:.=}])> \n\t",
+  "surroundingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    ["'", "'"],
+    ["\"", "\""]
+  ],
+  "folding": {
+    "markers": {
+      "start": "^\\s*//\\s*#?region\\b",
+      "end": "^\\s*//\\s*#?endregion\\b"
+    }
+  },
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+}

extensions/ql-vscode/media/dark/archive-plus.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M14.5 1H1.5L1 1.5V4.5L1.5 5H2V13.5L2.5 14H13.5L14 13.5V5H14.5L15 4.5V1.5L14.5 1ZM13.5 4H2.5H2V2H14V4H13.5ZM3 13V5H13V13H3ZM11 7H5V8H11V7Z" fill="#C5C5C5"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/dark/clear-all.svg

@@ -0,0 +1,7 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10 12.6L10.7 13.3L12.3 11.7L13.9 13.3L14.7 12.6L13 11L14.7 9.40005L13.9 8.60005L12.3 10.3L10.7 8.60005L10 9.40005L11.6 11L10 12.6Z" fill="#C5C5C5"/>
+<path d="M1 4L15 4L15 3L1 3L1 4Z" fill="#C5C5C5"/>
+<path d="M1 7L15 7L15 6L1 6L1 7Z" fill="#C5C5C5"/>
+<path d="M9 9.5L9 9L1 9L1 10L9 10L9 9.5Z" fill="#C5C5C5"/>
+<path d="M9 13L9 12.5L9 12L1 12L1 13L9 13Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/cloud-download.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9565 6H12.0064C12.8004 6 13.5618 6.31607 14.1232 6.87868C14.6846 7.44129 15 8.20435 15 9C15 9.79565 14.6846 10.5587 14.1232 11.1213C13.5618 11.6839 12.8004 12 12.0064 12V11C12.5357 11 13.0434 10.7893 13.4176 10.4142C13.7919 10.0391 14.0021 9.53044 14.0021 9C14.0021 8.46957 13.7919 7.96086 13.4176 7.58579C13.0434 7.21072 12.5357 7 12.0064 7H11.0924L10.9687 6.143C10.8938 5.60541 10.6456 5.10711 10.2618 4.72407C9.87801 4.34103 9.37977 4.09427 8.84303 4.02143C8.30629 3.94859 7.76051 4.05365 7.2889 4.3206C6.81729 4.58754 6.44573 5.00173 6.23087 5.5L5.89759 6.262L5.08933 6.073C4.90382 6.02699 4.71364 6.0025 4.52255 6C3.86093 6 3.22641 6.2634 2.75858 6.73224C2.29075 7.20108 2.02792 7.83696 2.02792 8.5C2.02792 9.16304 2.29075 9.79893 2.75858 10.2678C3.22641 10.7366 3.86093 11 4.52255 11H5.02148V12H4.52255C4.02745 12.0043 3.5371 11.903 3.08403 11.7029C2.63096 11.5028 2.22553 11.2084 1.89461 10.8394C1.5637 10.4703 1.31488 10.0349 1.16465 9.56211C1.01442 9.08932 0.966217 8.58992 1.02324 8.09704C1.08026 7.60416 1.24121 7.12906 1.4954 6.70326C1.74959 6.27745 2.09121 5.91068 2.49762 5.62727C2.90402 5.34385 3.36591 5.15027 3.85264 5.05937C4.33938 4.96847 4.83984 4.98232 5.32083 5.1C5.6241 4.40501 6.14511 3.82799 6.80496 3.45635C7.4648 3.08472 8.22753 2.9387 8.9776 3.04044C9.72768 3.14217 10.4242 3.4861 10.9618 4.02014C11.4993 4.55418 11.8485 5.24923 11.9565 6ZM6.70719 11.1214L8.0212 12.4354V7H9.01506V12.3992L10.2929 11.1214L11 11.8285L8.85356 13.9749H8.14645L6.00008 11.8285L6.70719 11.1214Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/edit.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M13.23 1H11.77L3.52002 9.25L3.35999 9.46997L1 13.59L2.41003 15L6.53003 12.64L6.75 12.48L15 4.22998V2.77002L13.23 1ZM2.41003 13.59L3.92004 10.59L5.37 12.04L2.41003 13.59ZM6.23999 11.53L4.46997 9.76001L12.47 1.76001L14.24 3.53003L6.23999 11.53Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/folder-opened-plus.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0.5 14H12.5L12.98 13.63L15.61 6.63L15.13 6H13V3.5L12.5 3H6.70996L5.84998 2.15002L5.5 2H0.5L0 2.5V13.5L0.5 14ZM1 3H5.29004L6.15002 3.84998L6.5 4H12V6H8.5L8.15002 6.15002L7.29004 7H2.5L2.03003 7.33997L1.03003 10.42L1 3ZM12.13 13H1.18994L2.85999 8H7.5L7.84998 7.84998L8.70996 7H14.5L12.13 13Z" fill="#C5C5C5"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/dark/lgtm-plus.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none">
+  <path d="M16.010 6.49c-3.885 0-7.167 0.906-9.328 2.813-0.063-0.12-0.109-0.219-0.188-0.339-0.224-0.365-0.438-0.776-1.104-1.188-0.411-0.26-0.87-0.438-1.349-0.516-0.208-0.021-0.422-0.021-0.63 0l0.135-0.016c-1.214 0-1.922 0.724-2.385 1.354-0.458 0.625-0.755 1.328-0.948 2.099-0.38 1.542-0.385 3.536 1.083 5.026 0.766 0.781 1.667 1.151 2.484 1.37 0.156 0.042 0.297 0.052 0.448 0.083 0.531 2.521 2.104 4.656 4.208 5.839v0.005c1.24 0.693 2.417 1.010 3.297 1.349 1.234 0.479 2.536 1 4.052 1.135l0.078 0.005h0.198c1.745 0 3.063-0.703 4.203-1.141 0.875-0.333 2.052-0.641 3.302-1.344 0.578-0.323 1.115-0.719 1.594-1.172 1.318-1.234 2.229-2.839 2.625-4.599 1.115-0.182 2.141-0.719 2.922-1.536 1.464-1.484 1.458-3.479 1.078-5.021-0.193-0.771-0.49-1.474-0.948-2.099-0.458-0.63-1.172-1.354-2.385-1.354l0.135 0.016c-0.208-0.021-0.422-0.021-0.63 0-0.479 0.078-0.938 0.255-1.344 0.516-0.667 0.411-0.88 0.823-1.104 1.182-0.073 0.12-0.12 0.219-0.188 0.333-2.156-1.901-5.432-2.802-9.313-2.802zM16.042 8.313c4.745 0 8.016 1.422 9.411 3.964 0.839-0.323 1.453-2.521 2.146-2.948 0.563-0.344 0.885-0.26 0.885-0.26 1.271 0 2.578 3.729 0.953 5.38-0.859 0.875-2.443 1.12-3.229 1.057-0.063 2.542-1.542 4.833-3.5 5.932-1 0.563-2.068 0.854-3.063 1.234-1.229 0.469-2.38 1.016-3.547 1.016h-0.125c-1.161-0.099-2.318-0.542-3.547-1.016-0.995-0.38-2.068-0.682-3.063-1.24-1.948-1.099-3.427-3.391-3.49-5.927-0.781 0.068-2.385-0.177-3.245-1.057-1.625-1.651-0.318-5.38 0.948-5.38 0 0 0.328-0.083 0.885 0.26 0.698 0.427 1.318 2.646 2.161 2.953 1.391-2.547 4.667-3.969 9.417-3.969zM10.875 11.422c-2.276-0.042-4.146 1.792-4.146 4.068 0 2.281 1.87 4.115 4.146 4.073 5.328-0.099 5.328-8.047 0-8.141zM21.208 11.422c-5.427 0-5.427 8.141 0 8.141s5.427-8.141 0-8.141zM11.453 13.708c2.349 0.063 2.349 3.552 0 3.615-1.182 0-2.042-1.115-1.75-2.255 0.318 0.771 1.469 0.547 1.464-0.292 0-0.406-0.318-0.745-0.729-0.76 0.302-0.203 0.656-0.313 1.016-0.307zM20.641 13.708c2.344 0.063 2.344 3.552 0 3.615-1.182 0-2.047-1.115-1.755-2.255 0.229 0.552 0.979 0.641 1.328 0.146 0.344-0.49 0.010-1.167-0.589-1.193 0.297-0.208 0.651-0.313 1.016-0.313zM15.359 19.906c-0.318 0.026-0.5 0.193-0.5 0.635 0 0.281 0.182 0.484 0.5 0.484 0.229 0 0.266-0.323 0.047-0.375-0.031-0.005-0.172-0.057-0.172-0.182 0-0.12 0-0.167 0.24-0.198 0.104-0.016 0.156-0.141 0.125-0.24s-0.125-0.135-0.24-0.125zM16.724 19.906c-0.115-0.005-0.208 0.026-0.24 0.125s0.021 0.224 0.125 0.24c0.24 0.031 0.24 0.078 0.24 0.198 0 0.125-0.141 0.177-0.172 0.182-0.219 0.052-0.182 0.375 0.042 0.375 0.323 0 0.51-0.203 0.51-0.484 0-0.443-0.188-0.609-0.505-0.635z" fill="#C5C5C5"/>
+  <line y2="24" x2="16" y1="26" x1="32" stroke-width="2" stroke="green" fill="none"/>
+  <line y2="16" x2="24" y1="32" x1="24" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/dark/preview.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M2 2H14L15 3V13L14 14H2L1 13V3L2 2ZM2 13H14V3H2V13ZM13 4H3V7H13V4ZM12 6H4V5H12V6ZM9 12H13V8H9V12ZM10 9H12V11H10V9ZM7 8H3V9H7V8ZM3 11H7V12H3V11Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/sort-alpha.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
+			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72
+				250.027,338.24 432,338.24 432,304.32" fill="#C5C5C5"/>
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
+			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
+				L113.28,140.48l41.387,110.507H71.893z" fill="#C5C5C5"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/dark/sort-date.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/trash.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10 3H12H13V4H12V13L11 14H4L3 13V4H2V3H5V2C5 1.73478 5.10531 1.48038 5.29285 1.29285C5.48038 1.10531 5.73478 1 6 1H9C9.26522 1 9.51962 1.10531 9.70715 1.29285C9.89469 1.48038 10 1.73478 10 2V3ZM9 2H6V3H9V2ZM4 13H11V4H4V13ZM6 5H5V12H6V5ZM7 5H8V12H7V5ZM9 5H10V12H9V5Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/light/archive-plus.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M14.5 1H1.5L1 1.5V4.5L1.5 5H2V13.5L2.5 14H13.5L14 13.5V5H14.5L15 4.5V1.5L14.5 1ZM13.5 4H2.5H2V2H14V4H13.5ZM3 13V5H13V13H3ZM11 7H5V8H11V7Z" fill="#424242"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/light/clear-all.svg

@@ -0,0 +1,7 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10.0001 12.6L10.7001 13.3L12.3001 11.7L13.9001 13.3L14.7001 12.6L13.0001 11L14.7001 9.40005L13.9001 8.60005L12.3001 10.3L10.7001 8.60005L10.0001 9.40005L11.6001 11L10.0001 12.6Z" fill="#424242"/>
+<path d="M1.00006 4L15.0001 4L15.0001 3L1.00006 3L1.00006 4Z" fill="#424242"/>
+<path d="M1.00006 7L15.0001 7L15.0001 6L1.00006 6L1.00006 7Z" fill="#424242"/>
+<path d="M9.00006 9.5L9.00006 9L1.00006 9L1.00006 10L9.00006 10L9.00006 9.5Z" fill="#424242"/>
+<path d="M9.00006 13L9.00006 12.5L9.00006 12L1.00006 12L1.00006 13L9.00006 13Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/cloud-download.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9565 6H12.0064C12.8004 6 13.5618 6.31607 14.1232 6.87868C14.6846 7.44129 15 8.20435 15 9C15 9.79565 14.6846 10.5587 14.1232 11.1213C13.5618 11.6839 12.8004 12 12.0064 12V11C12.5357 11 13.0434 10.7893 13.4176 10.4142C13.7919 10.0391 14.0021 9.53044 14.0021 9C14.0021 8.46957 13.7919 7.96086 13.4176 7.58579C13.0434 7.21072 12.5357 7 12.0064 7H11.0924L10.9687 6.143C10.8938 5.60541 10.6456 5.10711 10.2618 4.72407C9.87801 4.34103 9.37977 4.09427 8.84303 4.02143C8.30629 3.94859 7.76051 4.05365 7.2889 4.3206C6.81729 4.58754 6.44573 5.00173 6.23087 5.5L5.89759 6.262L5.08933 6.073C4.90382 6.02699 4.71364 6.0025 4.52255 6C3.86093 6 3.22641 6.2634 2.75858 6.73224C2.29075 7.20108 2.02792 7.83696 2.02792 8.5C2.02792 9.16304 2.29075 9.79893 2.75858 10.2678C3.22641 10.7366 3.86093 11 4.52255 11H5.02148V12H4.52255C4.02745 12.0043 3.5371 11.903 3.08403 11.7029C2.63096 11.5028 2.22553 11.2084 1.89461 10.8394C1.5637 10.4703 1.31488 10.0349 1.16465 9.56211C1.01442 9.08932 0.966217 8.58992 1.02324 8.09704C1.08026 7.60416 1.24121 7.12906 1.4954 6.70326C1.74959 6.27745 2.09121 5.91068 2.49762 5.62727C2.90402 5.34385 3.36591 5.15027 3.85264 5.05937C4.33938 4.96847 4.83984 4.98232 5.32083 5.1C5.6241 4.40501 6.14511 3.82799 6.80496 3.45635C7.4648 3.08472 8.22753 2.9387 8.9776 3.04044C9.72768 3.14217 10.4242 3.4861 10.9618 4.02014C11.4993 4.55418 11.8485 5.24923 11.9565 6ZM6.70719 11.1214L8.0212 12.4354V7H9.01506V12.3992L10.2929 11.1214L11 11.8285L8.85356 13.9749H8.14645L6.00008 11.8285L6.70719 11.1214Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/edit.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M13.2302 1H11.7703L3.52026 9.25L3.36023 9.46997L1.00024 13.59L2.41028 15L6.53027 12.64L6.75024 12.48L15.0002 4.22998V2.77002L13.2302 1ZM2.41028 13.59L3.92029 10.59L5.37024 12.04L2.41028 13.59ZM6.24023 11.53L4.47021 9.76001L12.4702 1.76001L14.2402 3.53003L6.24023 11.53Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/folder-opened-plus.svg

@@ -0,0 +1,12 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0)">
+<path d="M0.499817 14H12.4998L12.9798 13.63L15.6098 6.63L15.1298 6H12.9998V3.5L12.4998 3H6.70978L5.84979 2.15002L5.49982 2H0.499817L-0.000183105 2.5V13.5L0.499817 14ZM0.999817 3H5.28986L6.14984 3.84998L6.49982 4H11.9998V6H8.49982L8.14984 6.15002L7.28986 7H2.49982L2.02985 7.33997L1.02985 10.42L0.999817 3ZM12.1298 13H1.18976L2.8598 8H7.49982L7.84979 7.84998L8.70978 7H14.4998L12.1298 13Z" fill="#424242"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</g>
+<defs>
+<clipPath id="clip0">
+<path d="M-0.000183105 0H15.9998V16H-0.000183105V0Z" fill="white"/>
+</clipPath>
+</defs>
+</svg>

extensions/ql-vscode/media/light/lgtm-plus.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none">
+  <path d="M16.010 6.49c-3.885 0-7.167 0.906-9.328 2.813-0.063-0.12-0.109-0.219-0.188-0.339-0.224-0.365-0.438-0.776-1.104-1.188-0.411-0.26-0.87-0.438-1.349-0.516-0.208-0.021-0.422-0.021-0.63 0l0.135-0.016c-1.214 0-1.922 0.724-2.385 1.354-0.458 0.625-0.755 1.328-0.948 2.099-0.38 1.542-0.385 3.536 1.083 5.026 0.766 0.781 1.667 1.151 2.484 1.37 0.156 0.042 0.297 0.052 0.448 0.083 0.531 2.521 2.104 4.656 4.208 5.839v0.005c1.24 0.693 2.417 1.010 3.297 1.349 1.234 0.479 2.536 1 4.052 1.135l0.078 0.005h0.198c1.745 0 3.063-0.703 4.203-1.141 0.875-0.333 2.052-0.641 3.302-1.344 0.578-0.323 1.115-0.719 1.594-1.172 1.318-1.234 2.229-2.839 2.625-4.599 1.115-0.182 2.141-0.719 2.922-1.536 1.464-1.484 1.458-3.479 1.078-5.021-0.193-0.771-0.49-1.474-0.948-2.099-0.458-0.63-1.172-1.354-2.385-1.354l0.135 0.016c-0.208-0.021-0.422-0.021-0.63 0-0.479 0.078-0.938 0.255-1.344 0.516-0.667 0.411-0.88 0.823-1.104 1.182-0.073 0.12-0.12 0.219-0.188 0.333-2.156-1.901-5.432-2.802-9.313-2.802zM16.042 8.313c4.745 0 8.016 1.422 9.411 3.964 0.839-0.323 1.453-2.521 2.146-2.948 0.563-0.344 0.885-0.26 0.885-0.26 1.271 0 2.578 3.729 0.953 5.38-0.859 0.875-2.443 1.12-3.229 1.057-0.063 2.542-1.542 4.833-3.5 5.932-1 0.563-2.068 0.854-3.063 1.234-1.229 0.469-2.38 1.016-3.547 1.016h-0.125c-1.161-0.099-2.318-0.542-3.547-1.016-0.995-0.38-2.068-0.682-3.063-1.24-1.948-1.099-3.427-3.391-3.49-5.927-0.781 0.068-2.385-0.177-3.245-1.057-1.625-1.651-0.318-5.38 0.948-5.38 0 0 0.328-0.083 0.885 0.26 0.698 0.427 1.318 2.646 2.161 2.953 1.391-2.547 4.667-3.969 9.417-3.969zM10.875 11.422c-2.276-0.042-4.146 1.792-4.146 4.068 0 2.281 1.87 4.115 4.146 4.073 5.328-0.099 5.328-8.047 0-8.141zM21.208 11.422c-5.427 0-5.427 8.141 0 8.141s5.427-8.141 0-8.141zM11.453 13.708c2.349 0.063 2.349 3.552 0 3.615-1.182 0-2.042-1.115-1.75-2.255 0.318 0.771 1.469 0.547 1.464-0.292 0-0.406-0.318-0.745-0.729-0.76 0.302-0.203 0.656-0.313 1.016-0.307zM20.641 13.708c2.344 0.063 2.344 3.552 0 3.615-1.182 0-2.047-1.115-1.755-2.255 0.229 0.552 0.979 0.641 1.328 0.146 0.344-0.49 0.010-1.167-0.589-1.193 0.297-0.208 0.651-0.313 1.016-0.313zM15.359 19.906c-0.318 0.026-0.5 0.193-0.5 0.635 0 0.281 0.182 0.484 0.5 0.484 0.229 0 0.266-0.323 0.047-0.375-0.031-0.005-0.172-0.057-0.172-0.182 0-0.12 0-0.167 0.24-0.198 0.104-0.016 0.156-0.141 0.125-0.24s-0.125-0.135-0.24-0.125zM16.724 19.906c-0.115-0.005-0.208 0.026-0.24 0.125s0.021 0.224 0.125 0.24c0.24 0.031 0.24 0.078 0.24 0.198 0 0.125-0.141 0.177-0.172 0.182-0.219 0.052-0.182 0.375 0.042 0.375 0.323 0 0.51-0.203 0.51-0.484 0-0.443-0.188-0.609-0.505-0.635z" fill="#424242"/>
+  <line y2="24" x2="16" y1="26" x1="32" stroke-width="2" stroke="green" fill="none"/>
+  <line y2="16" x2="24" y1="32" x1="24" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/light/preview.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M2.00024 2H14.0002L15.0002 3V13L14.0002 14H2.00024L1.00024 13V3L2.00024 2ZM2.00024 13H14.0002V3H2.00024V13ZM13.0002 4H3.00024V7H13.0002V4ZM12.0002 6H4.00024V5H12.0002V6ZM9.00024 12H13.0002V8H9.00024V12ZM10.0002 9H12.0002V11H10.0002V9ZM7.00024 8H3.00024V9H7.00024V8ZM3.00024 11H7.00024V12H3.00024V11Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/sort-alpha.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413 			"/>
+			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72 
+				250.027,338.24 432,338.24 432,304.32 			"/>
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227 			"/>
+			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
+				L113.28,140.48l41.387,110.507H71.893z"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/light/sort-date.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/trash.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10.0002 3H12.0002H13.0002V4H12.0002V13L11.0002 14H4.00024L3.00024 13V4H2.00024V3H5.00024V2C5.00024 1.73478 5.10555 1.48038 5.29309 1.29285C5.48063 1.10531 5.73503 1 6.00024 1H9.00024C9.26546 1 9.51986 1.10531 9.7074 1.29285C9.89493 1.48038 10.0002 1.73478 10.0002 2V3ZM9.00024 2H6.00024V3H9.00024V2ZM4.00024 13H11.0002V4H4.00024V13ZM6.00024 5H5.00024V12H6.00024V5ZM7.00024 5H8.00024V12H7.00024V5ZM9.00024 5H10.0002V12H9.00024V5Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.0.2",
+  "version": "1.3.6",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,20 +13,35 @@
     "url": "https://github.com/github/vscode-codeql"
   },
   "engines": {
-    "vscode": "^1.39.0"
+    "vscode": "^1.43.0"
   },
   "categories": [
     "Programming Languages"
   ],
+  "extensionDependencies": [
+    "hbenl.vscode-test-explorer"
+  ],
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
+    "onView:codeQLAstViewer",
+    "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
-    "onCommand:codeQL.chooseDatabase",
+    "onCommand:codeQLDatabases.chooseDatabaseFolder",
+    "onCommand:codeQLDatabases.chooseDatabaseArchive",
+    "onCommand:codeQLDatabases.chooseDatabaseInternet",
+    "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
+    "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.chooseDatabaseFolder",
+    "onCommand:codeQL.chooseDatabaseArchive",
+    "onCommand:codeQL.chooseDatabaseInternet",
+    "onCommand:codeQL.chooseDatabaseLgtm",
     "onCommand:codeQLDatabases.chooseDatabase",
     "onCommand:codeQLDatabases.setCurrentDatabase",
+    "onCommand:codeQL.quickQuery",
+    "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
     "onFileSystem:codeql-zip-archive"
   ],
@@ -39,6 +54,14 @@
     "language-configuration.json"
   ],
   "contributes": {
+    "configurationDefaults": {
+      "[ql]": {
+        "editor.wordBasedSuggestions": false
+      },
+      "[dbscheme]": {
+        "editor.wordBasedSuggestions": false
+      }
+    },
     "languages": [
       {
         "id": "ql",
@@ -63,6 +86,12 @@
           ".dbscheme"
         ],
         "configuration": "./language-configuration.json"
+      },
+      {
+        "id": "xml",
+        "extensions": [
+          ".qhelp"
+        ]
       }
     ],
     "grammars": [
@@ -85,24 +114,30 @@
           "scope": "machine",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.cmd` on Windows. This overrides all other CodeQL CLI settings."
+          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
           "default": 1,
-          "minimum": 1,
+          "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running queries."
         },
         "codeQL.runningQueries.timeout": {
-          "type": ["integer", "null"],
+          "type": [
+            "integer",
+            "null"
+          ],
           "default": null,
           "minimum": 0,
           "maximum": 2147483647,
           "description": "Timeout (in seconds) for running queries. Leave blank or set to zero for no timeout."
         },
         "codeQL.runningQueries.memory": {
-          "type": ["integer", "null"],
+          "type": [
+            "integer",
+            "null"
+          ],
           "default": null,
           "minimum": 1024,
           "description": "Memory (in MB) to use for running queries. Leave blank for CodeQL to choose a suitable value based on your system's available memory."
@@ -112,10 +147,28 @@
           "default": false,
           "description": "Enable debug logging and tuple counting when running CodeQL queries. This information is useful for debugging query performance."
         },
+        "codeQL.runningQueries.autoSave": {
+          "type": "boolean",
+          "default": false,
+          "description": "Enable automatically saving a modified query file when running a query."
+        },
+        "codeQL.runningQueries.maxQueries": {
+          "type": "integer",
+          "default": 20,
+          "description": "Max number of simultaneous queries to run using the 'CodeQL: Run Queries' command."
+        },
         "codeQL.queryHistory.format": {
           "type": "string",
           "default": "[%t] %q on %d - %s",
           "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+        },
+        "codeQL.runningTests.numberOfThreads": {
+          "scope": "window",
+          "type": "integer",
+          "default": 1,
+          "minimum": 0,
+          "maximum": 1024,
+          "description": "Number of threads for running CodeQL tests."
         }
       }
     },
@@ -124,22 +177,58 @@
         "command": "codeQL.runQuery",
         "title": "CodeQL: Run Query"
       },
+      {
+        "command": "codeQL.runQueries",
+        "title": "CodeQL: Run Queries in Selected Files"
+      },
       {
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
       {
-        "command": "codeQL.chooseDatabase",
-        "title": "CodeQL: Choose Database",
+        "command": "codeQL.quickQuery",
+        "title": "CodeQL: Quick Query"
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseFolder",
+        "title": "Choose Database from Folder",
         "icon": {
-          "light": "media/black-plus.svg",
-          "dark": "media/white-plus.svg"
+          "light": "media/light/folder-opened-plus.svg",
+          "dark": "media/dark/folder-opened-plus.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseArchive",
+        "title": "Choose Database from Archive",
+        "icon": {
+          "light": "media/light/archive-plus.svg",
+          "dark": "media/dark/archive-plus.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseInternet",
+        "title": "Download Database",
+        "icon": {
+          "light": "media/light/cloud-download.svg",
+          "dark": "media/dark/cloud-download.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseLgtm",
+        "title": "Download from LGTM",
+        "icon": {
+          "light": "media/light/lgtm-plus.svg",
+          "dark": "media/dark/lgtm-plus.svg"
         }
       },
       {
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
       },
+      {
+        "command": "codeQL.viewAst",
+        "title": "CodeQL: View AST"
+      },
       {
         "command": "codeQL.upgradeCurrentDatabase",
         "title": "CodeQL: Upgrade Current Database"
@@ -160,21 +249,97 @@
         "command": "codeQLDatabases.upgradeDatabase",
         "title": "Upgrade Database"
       },
+      {
+        "command": "codeQLDatabases.renameDatabase",
+        "title": "Rename Database"
+      },
+      {
+        "command": "codeQLDatabases.openDatabaseFolder",
+        "title": "Show Database Directory"
+      },
+      {
+        "command": "codeQL.chooseDatabaseFolder",
+        "title": "CodeQL: Choose Database from Folder"
+      },
+      {
+        "command": "codeQL.chooseDatabaseArchive",
+        "title": "CodeQL: Choose Database from Archive"
+      },
+      {
+        "command": "codeQL.chooseDatabaseInternet",
+        "title": "CodeQL: Download Database"
+      },
+      {
+        "command": "codeQL.chooseDatabaseLgtm",
+        "title": "CodeQL: Download Database from LGTM"
+      },
+      {
+        "command": "codeQLDatabases.sortByName",
+        "title": "Sort by Name",
+        "icon": {
+          "light": "media/light/sort-alpha.svg",
+          "dark": "media/dark/sort-alpha.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.sortByDateAdded",
+        "title": "Sort by Date Added",
+        "icon": {
+          "light": "media/light/sort-date.svg",
+          "dark": "media/dark/sort-date.svg"
+        }
+      },
       {
         "command": "codeQL.checkForUpdatesToCLI",
         "title": "CodeQL: Check for CLI Updates"
       },
       {
         "command": "codeQLQueryHistory.openQuery",
-        "title": "Open Query"
-      },
-      {
-        "command": "codeQLQueryHistory.removeHistoryItem",
-        "title": "Remove History Item"
+        "title": "Open Query",
+        "icon": {
+          "light": "media/light/edit.svg",
+          "dark": "media/dark/edit.svg"
+        }
       },
       {
         "command": "codeQLQueryHistory.itemClicked",
-        "title": "Query History Item"
+        "title": "Open Query Results",
+        "icon": {
+          "light": "media/light/preview.svg",
+          "dark": "media/dark/preview.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.removeHistoryItem",
+        "title": "Remove History Item(s)",
+        "icon": {
+          "light": "media/light/trash.svg",
+          "dark": "media/dark/trash.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.showQueryLog",
+        "title": "Show Query Log"
+      },
+      {
+        "command": "codeQLQueryHistory.showQueryText",
+        "title": "Show Query Text"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarif",
+        "title": "View SARIF"
+      },
+      {
+        "command": "codeQLQueryHistory.viewDil",
+        "title": "View DIL"
+      },
+      {
+        "command": "codeQLQueryHistory.setLabel",
+        "title": "Set Label"
+      },
+      {
+        "command": "codeQLQueryHistory.compareWith",
+        "title": "Compare Results"
       },
       {
         "command": "codeQLQueryResults.nextPathStep",
@@ -185,16 +350,81 @@
         "title": "CodeQL: Show Previous Step on Path"
       },
       {
-        "command": "codeQLQueryHistory.setLabel",
-        "title": "Set Label"
+        "command": "codeQL.restartQueryServer",
+        "title": "CodeQL: Restart Query Server"
+      },
+      {
+        "command": "codeQLTests.showOutputDifferences",
+        "title": "Show Test Output Differences"
+      },
+      {
+        "command": "codeQLTests.acceptOutput",
+        "title": "Accept Test Output"
+      },
+      {
+        "command": "codeQLAstViewer.gotoCode",
+        "title": "Go To Code"
+      },
+      {
+        "command": "codeQLAstViewer.clear",
+        "title": "Clear AST",
+        "icon": {
+          "light": "media/light/clear-all.svg",
+          "dark": "media/dark/clear-all.svg"
+        }
       }
     ],
     "menus": {
       "view/title": [
         {
-          "command": "codeQL.chooseDatabase",
+          "command": "codeQLDatabases.sortByName",
           "when": "view == codeQLDatabases",
           "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.sortByDateAdded",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseFolder",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseArchive",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.openQuery",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.itemClicked",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.removeHistoryItem",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLAstViewer.clear",
+          "when": "view == codeQLAstViewer",
+          "group": "navigation"
         }
       ],
       "view/item/context": [
@@ -213,6 +443,16 @@
           "group": "9_qlCommands",
           "when": "view == codeQLDatabases"
         },
+        {
+          "command": "codeQLDatabases.renameDatabase",
+          "group": "9_qlCommands",
+          "when": "view == codeQLDatabases"
+        },
+        {
+          "command": "codeQLDatabases.openDatabaseFolder",
+          "group": "9_qlCommands",
+          "when": "view == codeQLDatabases"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "group": "9_qlCommands",
@@ -227,18 +467,57 @@
           "command": "codeQLQueryHistory.setLabel",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.compareWith",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryLog",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryText",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarif",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewDil",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "group": "qltest@1",
+          "when": "view == test-explorer && viewItem == testWithSource"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "group": "qltest@2",
+          "when": "view == test-explorer && viewItem == testWithSource"
         }
       ],
       "explorer/context": [
         {
           "command": "codeQL.setCurrentDatabase",
           "group": "9_qlCommands",
-          "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder"
+          "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
         },
         {
-          "command": "codeQL.runQuery",
+          "command": "codeQL.viewAst",
           "group": "9_qlCommands",
-          "when": "resourceLangId == ql && resourceExtname == .ql"
+          "when": "resourceScheme == codeql-zip-archive"
+        },
+        {
+          "command": "codeQL.runQueries",
+          "group": "9_qlCommands"
         }
       ],
       "commandPalette": [
@@ -246,6 +525,10 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueries",
+          "when": "false"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -254,14 +537,54 @@
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.renameDatabase",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.openDatabaseFolder",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.sortByName",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.sortByDateAdded",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.removeDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.chooseDatabaseFolder",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseArchive",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.upgradeDatabase",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "when": "false"
@@ -274,9 +597,45 @@
           "command": "codeQLQueryHistory.itemClicked",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.showQueryLog",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryText",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarif",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewDil",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.setLabel",
           "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.compareWith",
+          "when": "false"
+        },
+        {
+          "command": "codeQLAstViewer.gotoCode",
+          "when": "false"
+        },
+        {
+          "command": "codeQLAstViewer.clear",
+          "when": "false"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "when": "false"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -308,9 +667,27 @@
         {
           "id": "codeQLQueryHistory",
           "name": "Query History"
+        },
+        {
+          "id": "codeQLAstViewer",
+          "name": "AST Viewer"
         }
       ]
-    }
+    },
+    "viewsWelcome": [
+      {
+        "view": "codeQLAstViewer",
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+      },
+      {
+        "view": "codeQLQueryHistory",
+        "contents": "Run the 'CodeQL: Run Query' command on a QL query.\n[Run Query](command:codeQL.runQuery)"
+      },
+      {
+        "view": "codeQLDatabases",
+        "contents": "Add a Code QL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+      }
+    ]
   },
   "scripts": {
     "build": "gulp",
@@ -320,64 +697,106 @@
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
     "integration": "node ./out/vscode-tests/run-integration-tests.js",
     "update-vscode": "node ./node_modules/vscode/bin/install",
-    "postinstall": "node ./node_modules/vscode/bin/install",
-    "format": "tsfmt -r"
+    "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
+    "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
+    "format-staged": "lint-staged"
   },
   "dependencies": {
+    "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
     "fs-extra": "^8.1.0",
     "glob-promise": "^3.4.0",
+    "js-yaml": "^3.14.0",
+    "minimist": "~1.2.5",
     "node-fetch": "~2.6.0",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
-    "semmle-bqrs": "^0.0.1",
-    "semmle-io-node": "^0.0.1",
-    "semmle-vscode-utils": "^0.0.1",
+    "semver": "~7.3.2",
     "tmp": "^0.1.0",
+    "tmp-promise": "~3.0.2",
+    "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
-    "vscode-jsonrpc": "^4.0.0",
-    "vscode-languageclient": "^5.2.1"
+    "vscode-jsonrpc": "^5.0.1",
+    "vscode-languageclient": "^6.1.3",
+    "vscode-test-adapter-api": "~1.7.0",
+    "vscode-test-adapter-util": "~0.7.0",
+    "zip-a-folder": "~0.0.12"
   },
   "devDependencies": {
     "@types/chai": "^4.1.7",
+    "@types/chai-as-promised": "~7.1.2",
+    "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
     "@types/fs-extra": "^8.0.0",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
+    "@types/gulp-sourcemaps": "0.0.32",
+    "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
-    "@types/mocha": "~5.2.7",
+    "@types/mocha": "~8.0.3",
     "@types/node": "^12.0.8",
     "@types/node-fetch": "~2.5.2",
+    "@types/proxyquire": "~1.3.28",
     "@types/react": "^16.8.17",
     "@types/react-dom": "^16.8.4",
     "@types/sarif": "~2.1.2",
+    "@types/semver": "~7.2.0",
+    "@types/sinon": "~7.5.2",
+    "@types/sinon-chai": "~3.2.3",
+    "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.39.0",
+    "@types/vscode": "^1.43.0",
     "@types/webpack": "^4.32.1",
     "@types/xml2js": "~0.4.4",
-    "build-tasks": "^0.0.1",
+    "@typescript-eslint/eslint-plugin": "~2.23.0",
+    "@typescript-eslint/parser": "~2.23.0",
+    "ansi-colors": "^4.1.1",
     "chai": "^4.2.0",
-    "child-process-promise": "^2.2.1",
+    "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
+    "eslint": "~6.8.0",
+    "eslint-plugin-react": "~7.19.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
-    "mocha": "~6.2.1",
+    "husky": "~4.2.5",
+    "jsonc-parser": "^2.3.0",
+    "lint-staged": "~10.2.2",
+    "mocha": "~8.1.3",
+    "mocha-sinon": "~2.1.0",
     "npm-run-all": "^4.1.5",
+    "prettier": "~2.0.5",
+    "proxyquire": "~2.1.3",
+    "sinon": "~9.0.0",
+    "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
     "ts-loader": "^5.4.5",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "^3.7.2",
-    "typescript-config": "^0.0.1",
+    "typescript": "~3.8.3",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
-    "vscode-test": "^1.0.0",
+    "vscode-test": "^1.4.0",
     "webpack": "^4.38.0",
     "webpack-cli": "^3.3.2"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "npm run format-staged",
+      "pre-push": "npm run lint"
+    }
+  },
+  "lint-staged": {
+    "./**/*.{json,css,scss,md}": [
+      "prettier --write"
+    ],
+    "./**/*.{ts,tsx}": [
+      "tsfmt -r",
+      "eslint --fix"
+    ]
   }
 }

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -51,7 +51,10 @@ export type Entry = File | Directory;
  */
 export type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
 
-export type ZipFileReference = { sourceArchiveZipPath: string, pathWithinSourceArchive: string };
+export type ZipFileReference = {
+  sourceArchiveZipPath: string;
+  pathWithinSourceArchive: string;
+};
 
 /** Encodes a reference to a source file within a zipped source archive into a single URI. */
 export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
@@ -81,13 +84,26 @@ export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
   // This lets us separate the paths, ignoring the leading slash if we added one.
   const sourceArchiveZipPathEndIndex = sourceArchiveZipPathStartIndex + sourceArchiveZipPath.length;
   const authority = `${sourceArchiveZipPathStartIndex}-${sourceArchiveZipPathEndIndex}`;
-  return vscode.Uri.parse(zipArchiveScheme + ':/').with({
+  return vscode.Uri.parse(zipArchiveScheme + ':/', true).with({
     path: encodedPath,
     authority,
   });
 }
 
-const sourceArchiveUriAuthorityPattern = /^(\d+)\-(\d+)$/;
+/**
+ * Convenience method to create a codeql-zip-archive with a path to the root
+ * archive
+ *
+ * @param pathToArchive the filesystem path to the root of the archive
+ */
+export function encodeArchiveBasePath(sourceArchiveZipPath: string) {
+  return encodeSourceArchiveUri({
+    sourceArchiveZipPath,
+    pathWithinSourceArchive: ''
+  });
+}
+
+const sourceArchiveUriAuthorityPattern = /^(\d+)-(\d+)$/;
 
 class InvalidSourceArchiveUriError extends Error {
   constructor(uri: vscode.Uri) {
@@ -97,6 +113,14 @@ class InvalidSourceArchiveUriError extends Error {
 
 /** Decodes an encoded source archive URI into its corresponding paths. Inverse of `encodeSourceArchiveUri`. */
 export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
+  if (!uri.authority) {
+    // Uri is malformed, but this is recoverable
+    logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
+    return {
+      pathWithinSourceArchive: '/',
+      sourceArchiveZipPath: uri.path
+    };
+  }
   const match = sourceArchiveUriAuthorityPattern.exec(uri.authority);
   if (match === null)
     throw new InvalidSourceArchiveUriError(uri);
@@ -105,7 +129,7 @@ export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
   if (isNaN(zipPathStartIndex) || isNaN(zipPathEndIndex))
     throw new InvalidSourceArchiveUriError(uri);
   return {
-    pathWithinSourceArchive: uri.path.substring(zipPathEndIndex),
+    pathWithinSourceArchive: uri.path.substring(zipPathEndIndex) || '/',
     sourceArchiveZipPath: uri.path.substring(zipPathStartIndex, zipPathEndIndex),
   };
 }
@@ -139,8 +163,8 @@ function ensureDir(map: DirectoryHierarchyMap, dir: string) {
 }
 
 type Archive = {
-  unzipped: unzipper.CentralDirectory,
-  dirMap: DirectoryHierarchyMap,
+  unzipped: unzipper.CentralDirectory;
+  dirMap: DirectoryHierarchyMap;
 };
 
 export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
@@ -169,8 +193,8 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
   async readDirectory(uri: vscode.Uri): Promise<[string, vscode.FileType][]> {
     const ref = decodeSourceArchiveUri(uri);
     const archive = await this.getArchive(ref.sourceArchiveZipPath);
-    let contents = archive.dirMap.get(ref.pathWithinSourceArchive);
-    const result = contents === undefined ? [] : Array.from(contents.entries());
+    const contents = archive.dirMap.get(ref.pathWithinSourceArchive);
+    const result = contents === undefined ? undefined : Array.from(contents.entries());
     if (result === undefined) {
       throw vscode.FileSystemError.FileNotFound(uri);
     }
@@ -189,7 +213,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
 
   // write operations, all disabled
 
-  writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { create: boolean, overwrite: boolean }): void {
+  writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { create: boolean; overwrite: boolean }): void {
     throw this.readOnlyError;
   }
 
@@ -235,11 +259,11 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
     if (archive.dirMap.has(reqPath)) {
       return new Directory(reqPath);
     }
-    throw vscode.FileSystemError.FileNotFound(uri);
+    throw vscode.FileSystemError.FileNotFound(`uri '${uri.toString()}', interpreted as '${reqPath}' in archive '${ref.sourceArchiveZipPath}'`);
   }
 
   private async _lookupAsFile(uri: vscode.Uri): Promise<File> {
-    let entry = await this._lookup(uri);
+    const entry = await this._lookup(uri);
     if (entry instanceof File) {
       return entry;
     }
@@ -254,7 +278,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
 
   watch(_resource: vscode.Uri): vscode.Disposable {
     // ignore, fires for all changes...
-    return new vscode.Disposable(() => { });
+    return new vscode.Disposable(() => { /**/ });
   }
 }
 

extensions/ql-vscode/src/astViewer.ts

@@ -0,0 +1,193 @@
+import {
+  window,
+  TreeDataProvider,
+  EventEmitter,
+  Event,
+  ProviderResult,
+  TreeItemCollapsibleState,
+  TreeItem,
+  TreeView,
+  TextEditorSelectionChangeEvent,
+  TextEditorSelectionChangeKind,
+  Location,
+  Range
+} from 'vscode';
+import * as path from 'path';
+
+import { DatabaseItem } from './databases';
+import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
+import { showLocation } from './interface-utils';
+import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
+import { commandRunner } from './helpers';
+import { DisposableObject } from './vscode-utils/disposable-object';
+
+export interface AstItem {
+  id: BqrsId;
+  label?: string;
+  location?: UrlValue;
+  fileLocation?: Location;
+  children: ChildAstItem[];
+  order: number;
+}
+
+export interface ChildAstItem extends AstItem {
+  parent: ChildAstItem | AstItem;
+}
+
+class AstViewerDataProvider extends DisposableObject implements TreeDataProvider<AstItem> {
+
+  public roots: AstItem[] = [];
+  public db: DatabaseItem | undefined;
+
+  private _onDidChangeTreeData =
+    new EventEmitter<AstItem | undefined>();
+  readonly onDidChangeTreeData: Event<AstItem | undefined> =
+    this._onDidChangeTreeData.event;
+
+  constructor() {
+    super();
+    this.push(
+      commandRunner('codeQLAstViewer.gotoCode',
+        async (item: AstItem) => {
+          await showLocation(item.fileLocation);
+        })
+    );
+  }
+
+  refresh(): void {
+    this._onDidChangeTreeData.fire();
+  }
+  getChildren(item?: AstItem): ProviderResult<AstItem[]> {
+    const children = item ? item.children : this.roots;
+    return children.sort((c1, c2) => (c1.order - c2.order));
+  }
+
+  getParent(item: ChildAstItem): ProviderResult<AstItem> {
+    return item.parent;
+  }
+
+  getTreeItem(item: AstItem): TreeItem {
+    const line = this.extractLineInfo(item?.location);
+
+    const state = item.children.length
+      ? TreeItemCollapsibleState.Collapsed
+      : TreeItemCollapsibleState.None;
+    const treeItem = new TreeItem(item.label || '', state);
+    treeItem.description = line ? `Line ${line}` : '';
+    treeItem.id = String(item.id);
+    treeItem.tooltip = `${treeItem.description} ${treeItem.label}`;
+    treeItem.command = {
+      command: 'codeQLAstViewer.gotoCode',
+      title: 'Go To Code',
+      tooltip: `Go To ${item.location}`,
+      arguments: [item]
+    };
+    return treeItem;
+  }
+
+  private extractLineInfo(loc?: UrlValue) {
+    if (!loc) {
+      return '';
+    } else if (isStringLoc(loc)) {
+      return loc;
+    } else if (isWholeFileLoc(loc)) {
+      return loc.uri;
+    } else if (isLineColumnLoc(loc)) {
+      return loc.startLine;
+    } else {
+      return '';
+    }
+  }
+}
+
+export class AstViewer extends DisposableObject {
+  private treeView: TreeView<AstItem>;
+  private treeDataProvider: AstViewerDataProvider;
+  private currentFile: string | undefined;
+
+  constructor() {
+    super();
+
+    this.treeDataProvider = new AstViewerDataProvider();
+    this.treeView = window.createTreeView('codeQLAstViewer', {
+      treeDataProvider: this.treeDataProvider,
+      showCollapseAll: true
+    });
+
+    this.push(this.treeView);
+    this.push(this.treeDataProvider);
+    this.push(
+      commandRunner('codeQLAstViewer.clear', async () => {
+        this.clear();
+      })
+    );
+    this.push(window.onDidChangeTextEditorSelection(this.updateTreeSelection, this));
+  }
+
+  updateRoots(roots: AstItem[], db: DatabaseItem, fileName: string) {
+    this.treeDataProvider.roots = roots;
+    this.treeDataProvider.db = db;
+    this.treeDataProvider.refresh();
+    this.treeView.message = `AST for ${path.basename(fileName)}`;
+    this.treeView.reveal(roots[0], { focus: false });
+    this.currentFile = fileName;
+  }
+
+  private updateTreeSelection(e: TextEditorSelectionChangeEvent) {
+    function isInside(selectedRange: Range, astRange?: Range): boolean {
+      return !!astRange?.contains(selectedRange);
+    }
+
+    // Recursively iterate all children until we find the node with the smallest
+    // range that contains the selection.
+    // Some nodes do not have a location, but their children might, so must
+    // recurse though location-less AST nodes to see if children are correct.
+    function findBest(selectedRange: Range, items?: AstItem[]): AstItem | undefined {
+      if (!items || !items.length) {
+        return;
+      }
+      for (const item of items) {
+        let candidate: AstItem | undefined = undefined;
+        if (isInside(selectedRange, item.fileLocation?.range)) {
+          candidate = item;
+        }
+        // always iterate through children since the location of an AST node in code QL does not
+        // always cover the complete text of the node.
+        candidate = findBest(selectedRange, item.children) || candidate;
+        if (candidate) {
+          return candidate;
+        }
+      }
+      return;
+    }
+
+    // Avoid recursive tree-source code updates.
+    if (e.kind === TextEditorSelectionChangeKind.Command) {
+      return;
+    }
+
+    if (
+      this.treeView.visible &&
+      e.textEditor.document.uri.fsPath === this.currentFile &&
+      e.selections.length === 1
+    ) {
+      const selection = e.selections[0];
+      const range = selection.anchor.isBefore(selection.active)
+        ? new Range(selection.anchor, selection.active)
+        : new Range(selection.active, selection.anchor);
+
+      const targetItem = findBest(range, this.treeDataProvider.roots);
+      if (targetItem) {
+        this.treeView.reveal(targetItem);
+      }
+    }
+  }
+
+  private clear() {
+    this.treeDataProvider.roots = [];
+    this.treeDataProvider.db = undefined;
+    this.treeDataProvider.refresh();
+    this.treeView.message = undefined;
+    this.currentFile = undefined;
+  }
+}

extensions/ql-vscode/src/cli-version.ts

@@ -1,96 +1,24 @@
-import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "./logging";
+import * as semver from 'semver';
+import { runCodeQlCliCommand } from './cli';
+import { Logger } from './logging';
 
 /**
  * Get the version of a CodeQL CLI.
  */
-export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<Version | undefined> {
-  const output: string = await runCodeQlCliCommand(
-    codeQlPath,
-    ["version"],
-    ["--format=terse"],
-    "Checking CodeQL version",
-    logger
-  );
-  return tryParseVersionString(output.trim());
-}
-
-/**
- * Try to parse a version string, returning undefined if we can't parse it.
- * 
- * Version strings must contain a major, minor, and patch version.  They may optionally
- * start with "v" and may optionally contain some "tail" string after the major, minor, and
- * patch versions, for example as in `v2.1.0+baf5bff`.
- */
-export function tryParseVersionString(versionString: string): Version | undefined {
-  const match = versionString.match(versionRegex);
-  if (match === null) {
+export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<semver.SemVer | undefined> {
+  try {
+    const output: string = await runCodeQlCliCommand(
+      codeQlPath,
+      ['version'],
+      ['--format=terse'],
+      'Checking CodeQL version',
+      logger
+    );
+    return semver.parse(output.trim()) || undefined;
+  } catch (e) {
+    // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
+    // Either way, we can't determine compatibility.
+    logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
     return undefined;
   }
-  return {
-    buildMetadata: match[5],
-    majorVersion: Number.parseInt(match[1], 10),
-    minorVersion: Number.parseInt(match[2], 10),
-    patchVersion: Number.parseInt(match[3], 10),
-    prereleaseVersion: match[4],
-    rawString: versionString,
-  }
-}
-
-/**
- * Regex for parsing semantic versions
- * 
- * From the semver spec https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
- */
-const versionRegex = new RegExp(String.raw`^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)` +
-  String.raw`(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?` +
-  String.raw`(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`);
-
-/**
- * A version of the CodeQL CLI.
- */
-export interface Version {
-  /**
-   * Build metadata
-   * 
-   * For example, this will be `abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   * Build metadata must be ignored when comparing versions. 
-   */
-  buildMetadata: string | undefined;
-
-  /**
-   * Major version number
-   * 
-   * For example, this will be `2` for version 2.1.0-alpha.1+abcdef0.
-   */
-  majorVersion: number;
-
-  /**
-   * Minor version number
-   * 
-   * For example, this will be `1` for version 2.1.0-alpha.1+abcdef0.
-   */
-  minorVersion: number;
-
-  /**
-   * Patch version number
-   * 
-   * For example, this will be `0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  patchVersion: number;
-
-  /**
-   * Prerelease version
-   * 
-   * For example, this will be `alpha.1` for version 2.1.0-alpha.1+abcdef0.
-   * The prerelease version must be considered when comparing versions.
-   */
-  prereleaseVersion: string | undefined;
-
-  /**
-   * Raw version string
-   * 
-   * For example, this will be `2.1.0-alpha.1+abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  rawString: string;
 }

extensions/ql-vscode/src/cli.ts

@@ -1,18 +1,27 @@
-import * as child_process from "child_process";
+/* eslint-disable @typescript-eslint/camelcase */
+import * as cpp from 'child-process-promise';
+import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
-import * as util from 'util';
-import { Logger, ProgressReporter } from "./logging";
-import { Disposable } from "vscode";
-import { DistributionProvider } from "./distribution";
-import { SortDirection } from "./interface-types";
-import { assertNever } from "./helpers-pure";
+import { SemVer } from 'semver';
+import { Readable } from 'stream';
+import { StringDecoder } from 'string_decoder';
+import * as tk from 'tree-kill';
+import { promisify } from 'util';
+import { CancellationToken, Disposable } from 'vscode';
+
+import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
+import { CliConfig } from './config';
+import { DistributionProvider, FindDistributionResultKind } from './distribution';
+import { assertNever } from './pure/helpers-pure';
+import { QueryMetadata, SortDirection } from './pure/interface-types';
+import { Logger, ProgressReporter } from './logging';
 
 /**
  * The version of the SARIF format that we are using.
  */
-const SARIF_FORMAT = "sarifv2.1.0";
+const SARIF_FORMAT = 'sarifv2.1.0';
 
 /**
  * Flags to pass to all cli commands.
@@ -23,10 +32,10 @@ const LOGGING_FLAGS = ['-v', '--log-to-stderr'];
  * The expected output of `codeql resolve library-path`.
  */
 export interface QuerySetup {
-  libraryPath: string[],
-  dbscheme: string,
-  relativeName?: string,
-  compilationCache?: string
+  libraryPath: string[];
+  dbscheme: string;
+  relativeName?: string;
+  compilationCache?: string;
 }
 
 /**
@@ -51,14 +60,9 @@ export interface UpgradesInfo {
 }
 
 /**
- * The expected output of `codeql resolve metadata`.
+ * The expected output of `codeql resolve qlpacks`.
  */
-export interface QueryMetadata {
-  name?: string,
-  description?: string,
-  id?: string,
-  kind?: string
-}
+export type QlpacksInfo = { [name: string]: string[] };
 
 // `codeql bqrs interpret` requires both of these to be present or
 // both absent.
@@ -67,6 +71,43 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve tests`.
+ */
+export type ResolvedTests = string[];
+
+/**
+ * Options for `codeql test run`.
+ */
+export interface TestRunOptions {
+  cancellationToken?: CancellationToken;
+  logger?: Logger;
+}
+
+/**
+ * Event fired by `codeql test run`.
+ */
+export interface TestCompleted {
+  test: string;
+  pass: boolean;
+  messages: string[];
+  compilationMs: number;
+  evaluationMs: number;
+  expected: string;
+}
+
+/**
+ * Optional arguments for the `bqrsDecode` function
+ */
+interface BqrsDecodeOptions {
+  /** How many results to get. */
+  pageSize?: number;
+  /** The 0-based index of the first result to get. */
+  offset?: number;
+  /** The entity names to retrieve from the bqrs file. Default is url, string */
+  entities?: string[];
+}
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -75,6 +116,11 @@ export interface SourceInfo {
  */
 export class CodeQLCliServer implements Disposable {
 
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  private static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
+
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
   /** Queue of future commands*/
@@ -84,28 +130,43 @@ export class CodeQLCliServer implements Disposable {
   /**  A buffer with a single null byte. */
   nullBuffer: Buffer;
 
-  constructor(private config: DistributionProvider, private logger: Logger) {
+  /** Version of current cli, lazily computed by the `getVersion()` method */
+  _version: SemVer | undefined;
+
+  /** Path to current codeQL executable, or undefined if not running yet. */
+  codeQlPath: string | undefined;
+
+  constructor(
+    private distributionProvider: DistributionProvider,
+    private cliConfig: CliConfig,
+    private logger: Logger,
+  ) {
     this.commandQueue = [];
     this.commandInProcess = false;
     this.nullBuffer = Buffer.alloc(1);
-    if (this.config.onDidChangeDistribution) {
-      this.config.onDidChangeDistribution(() => {
+    if (this.distributionProvider.onDidChangeDistribution) {
+      this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
       });
     }
+    if (this.cliConfig.onDidChangeConfiguration) {
+      this.cliConfig.onDidChangeConfiguration(() => {
+        this.restartCliServer();
+        this._version = undefined;
+      });
+    }
   }
 
-
-  dispose() {
+  dispose(): void {
     this.killProcessIfRunning();
   }
 
-  killProcessIfRunning() {
+  killProcessIfRunning(): void {
     if (this.process) {
       // Tell the Java CLI server process to shut down.
       this.logger.log('Sending shutdown request');
       try {
-        this.process.stdin.write(JSON.stringify(["shutdown"]), "utf8");
+        this.process.stdin.write(JSON.stringify(['shutdown']), 'utf8');
         this.process.stdin.write(this.nullBuffer);
         this.logger.log('Sent shutdown request');
       } catch (e) {
@@ -127,8 +188,8 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Restart the server when the current command terminates
    */
-  private restartCliServer() {
-    let callback = () => {
+  private restartCliServer(): void {
+    const callback = (): void => {
       try {
         this.killProcessIfRunning();
       } finally {
@@ -139,43 +200,58 @@ export class CodeQLCliServer implements Disposable {
     // If the server is not running a command run this immediately
     // otherwise add to the front of the queue (as we want to run this after the next command()).
     if (this.commandInProcess) {
-      this.commandQueue.unshift(callback)
+      this.commandQueue.unshift(callback);
     } else {
       callback();
     }
 
   }
 
+  /**
+   * Get the path to the CodeQL CLI distribution, or throw an exception if not found.
+   */
+  private async getCodeQlPath(): Promise<string> {
+    const codeqlPath = await this.distributionProvider.getCodeQlPathWithoutVersionCheck();
+    if (!codeqlPath) {
+      throw new Error('Failed to find CodeQL distribution.');
+    }
+    return codeqlPath;
+  }
+
   /**
    * Launch the cli server
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
-    const config = await this.config.getCodeQlPathWithoutVersionCheck();
-    if (!config) {
-      throw new Error("Failed to find codeql distribution")
-    }
-    return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, _data => {})
+    const codeQlPath = await this.getCodeQlPath();
+    return await spawnServer(
+      codeQlPath,
+      'CodeQL CLI Server',
+      ['execute', 'cli-server'],
+      [],
+      this.logger,
+      _data => { /**/ }
+    );
   }
 
   private async runCodeQlCliInternal(command: string[], commandArgs: string[], description: string): Promise<string> {
-    let stderrBuffers: Buffer[] = [];
+    const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
-      throw new Error("runCodeQlCliInternal called while cli was running")
+      throw new Error('runCodeQlCliInternal called while cli was running');
     }
     this.commandInProcess = true;
     try {
       //Launch the process if it doesn't exist
       if (!this.process) {
-        this.process = await this.launchProcess()
+        this.process = await this.launchProcess();
       }
       // Grab the process so that typescript know that it is always defined.
       const process = this.process;
       // The array of fragments of stdout
-      let stdoutBuffers: Buffer[] = [];
+      const stdoutBuffers: Buffer[] = [];
 
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
-      const argsString = args.join(" ");
+      const argsString = args.join(' ');
       this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
         await new Promise((resolve, reject) => {
@@ -194,32 +270,33 @@ export class CodeQLCliServer implements Disposable {
             stderrBuffers.push(newData);
           });
           // Listen for process exit.
-          process.addListener("close", (code) => reject(code));
+          process.addListener('close', (code) => reject(code));
           // Write the command followed by a null terminator.
-          process.stdin.write(JSON.stringify(args), "utf8")
-          process.stdin.write(this.nullBuffer)
+          process.stdin.write(JSON.stringify(args), 'utf8');
+          process.stdin.write(this.nullBuffer);
         });
         // Join all the data together
-        let fullBuffer = Buffer.concat(stdoutBuffers);
+        const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
-        let data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        this.logger.log(`CLI command succeeded.`);
+        const data = fullBuffer.toString('utf8', 0, fullBuffer.length - 1);
+        this.logger.log('CLI command succeeded.');
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
         // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
-        if (stderrBuffers.length == 0) {
-          throw new Error(`${description} failed: ${err}`)
-        } else {
-          throw new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString("utf8")}`);
-        }
+        const newError =
+          stderrBuffers.length == 0
+            ? new Error(`${description} failed: ${err}`)
+            : new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString('utf8')}`);
+        newError.stack += (err.stack || '');
+        throw newError;
       } finally {
-        this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
         // Remove the listeners we set up.
-        process.stdout.removeAllListeners('data')
-        process.stderr.removeAllListeners('data')
-        process.removeAllListeners("close");
+        process.stdout.removeAllListeners('data');
+        process.stderr.removeAllListeners('data');
+        process.removeAllListeners('close');
       }
     } finally {
       this.commandInProcess = false;
@@ -231,13 +308,94 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Run the next command in the queue
    */
-  private runNext() {
+  private runNext(): void {
     const callback = this.commandQueue.shift();
     if (callback) {
       callback();
     }
   }
 
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param cancellationToken CancellationToken to terminate the test process.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  private async* runAsyncCodeQlCliCommandInternal(
+    command: string[],
+    commandArgs: string[],
+    cancellationToken?: CancellationToken,
+    logger?: Logger
+  ): AsyncGenerator<string, void, unknown> {
+    // Add format argument first, in case commandArgs contains positional parameters.
+    const args = [
+      ...command,
+      '--format', 'jsonz',
+      ...commandArgs
+    ];
+
+    // Spawn the CodeQL process
+    const codeqlPath = await this.getCodeQlPath();
+    const childPromise = cpp.spawn(codeqlPath, args);
+    const child = childPromise.childProcess;
+
+    let cancellationRegistration: Disposable | undefined = undefined;
+    try {
+      if (cancellationToken !== undefined) {
+        cancellationRegistration = cancellationToken.onCancellationRequested(_e => {
+          tk(child.pid);
+        });
+      }
+      if (logger !== undefined) {
+        // The human-readable output goes to stderr.
+        logStream(child.stderr!, logger);
+      }
+
+      for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
+        yield event;
+      }
+
+      await childPromise;
+    }
+    finally {
+      if (cancellationRegistration !== undefined) {
+        cancellationRegistration.dispose();
+      }
+    }
+  }
+
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param description Description of the action being run, to be shown in log and error messages.
+   * @param cancellationToken CancellationToken to terminate the test process.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  public async* runAsyncCodeQlCliCommand<EventType>(
+    command: string[],
+    commandArgs: string[],
+    description: string,
+    cancellationToken?: CancellationToken,
+    logger?: Logger
+  ): AsyncGenerator<EventType, void, unknown> {
+    for await (const event of await this.runAsyncCodeQlCliCommandInternal(command, commandArgs,
+      cancellationToken, logger)) {
+      try {
+        yield JSON.parse(event) as EventType;
+      } catch (err) {
+        throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`);
+      }
+    }
+  }
+
   /**
    * Runs a CodeQL CLI command on the server, returning the output as a string.
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
@@ -253,17 +411,17 @@ export class CodeQLCliServer implements Disposable {
 
     return new Promise((resolve, reject) => {
       // Construct the command that actually does the work
-      const callback = () => {
+      const callback = (): void => {
         try {
           this.runCodeQlCliInternal(command, commandArgs, description).then(resolve, reject);
         } catch (err) {
           reject(err);
         }
-      }
+      };
       // If the server is not running a command, then run the given command immediately,
       // otherwise add to the queue
       if (this.commandInProcess) {
-        this.commandQueue.push(callback)
+        this.commandQueue.push(callback);
       } else {
         callback();
       }
@@ -285,7 +443,7 @@ export class CodeQLCliServer implements Disposable {
     try {
       return JSON.parse(result) as OutputType;
     } catch (err) {
-      throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`)
+      throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`);
     }
   }
 
@@ -297,10 +455,45 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(workspaces: string[], queryPath: string): Promise<QuerySetup> {
     const subcommandArgs = [
       '--query', queryPath,
-      "--additional-packs",
+      '--additional-packs',
       workspaces.join(path.delimiter)
     ];
-    return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, "Resolving library paths");
+    return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, 'Resolving library paths');
+  }
+
+  /**
+   * Finds all available QL tests in a given directory.
+   * @param testPath Root of directory tree to search for tests.
+   * @returns The list of tests that were found.
+   */
+  public async resolveTests(testPath: string): Promise<ResolvedTests> {
+    const subcommandArgs = [
+      testPath
+    ];
+    return await this.runJsonCodeQlCliCommand<ResolvedTests>(['resolve', 'tests'], subcommandArgs, 'Resolving tests');
+  }
+
+  /**
+   * Runs QL tests.
+   * @param testPaths Full paths of the tests to run.
+   * @param workspaces Workspace paths to use as search paths for QL packs.
+   * @param options Additional options.
+   */
+  public async* runTests(
+    testPaths: string[], workspaces: string[], options: TestRunOptions
+  ): AsyncGenerator<TestCompleted, void, unknown> {
+
+    const subcommandArgs = [
+      '--additional-packs', workspaces.join(path.delimiter),
+      '--threads',
+      this.cliConfig.numberTestThreads.toString(),
+      ...testPaths
+    ];
+
+    for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
+      subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
+      yield event;
+    }
   }
 
   /**
@@ -308,13 +501,14 @@ export class CodeQLCliServer implements Disposable {
    * @param queryPath The path to the query.
    */
   async resolveMetadata(queryPath: string): Promise<QueryMetadata> {
-    return await this.runJsonCodeQlCliCommand<QueryMetadata>(['resolve', 'metadata'], [queryPath], "Resolving query metadata");
+    return await this.runJsonCodeQlCliCommand<QueryMetadata>(['resolve', 'metadata'], [queryPath], 'Resolving query metadata');
   }
 
   /**
    * Gets the RAM setting for the query server.
    * @param queryMemoryMb The maximum amount of RAM to use, in MB.
    * Leave `undefined` for CodeQL to choose a limit based on the available system memory.
+   * @param progressReporter The progress reporter to send progress information to.
    * @returns String arguments that can be passed to the CodeQL query server,
    * indicating how to split the given RAM limit between heap and off-heap memory.
    */
@@ -323,41 +517,76 @@ export class CodeQLCliServer implements Disposable {
     if (queryMemoryMb !== undefined) {
       args.push('--ram', queryMemoryMb.toString());
     }
-    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, "Resolving RAM settings", progressReporter);
+    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', progressReporter);
+  }
+  /**
+   * Gets the headers (and optionally pagination info) of a bqrs.
+   * @param bqrsPath The path to the bqrs.
+   * @param pageSize The page size to precompute offsets into the binary file for.
+   */
+  async bqrsInfo(bqrsPath: string, pageSize?: number): Promise<BQRSInfo> {
+    const subcommandArgs = (
+      pageSize ? ['--paginate-rows', pageSize.toString()] : []
+    ).concat(
+      bqrsPath
+    );
+    return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, 'Reading bqrs header');
   }
 
+  /**
+  * Gets the results from a bqrs.
+  * @param bqrsPath The path to the bqrs.
+  * @param resultSet The result set to get.
+  * @param options Optional BqrsDecodeOptions arguments
+  */
+  async bqrsDecode(
+    bqrsPath: string,
+    resultSet: string,
+    { pageSize, offset, entities = ['url', 'string'] }: BqrsDecodeOptions = {}
+  ): Promise<DecodedBqrsChunk> {
 
-  async interpretBqrs(metadata: { kind: string, id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    const subcommandArgs = [
+      `--entities=${entities.join(',')}`,
+      '--result-set', resultSet,
+    ].concat(
+      pageSize ? ['--rows', pageSize.toString()] : []
+    ).concat(
+      offset ? ['--start-at', offset.toString()] : []
+    ).concat([bqrsPath]);
+    return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
+  }
+
+  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     const args = [
       `-t=kind=${metadata.kind}`,
       `-t=id=${metadata.id}`,
-      "--output", interpretedResultsPath,
-      "--format", SARIF_FORMAT,
+      '--output', interpretedResultsPath,
+      '--format', SARIF_FORMAT,
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      "--no-group-results",
+      '--no-group-results',
     ];
     if (sourceInfo !== undefined) {
       args.push(
-        "--source-archive", sourceInfo.sourceArchive,
-        "--source-location-prefix", sourceInfo.sourceLocationPrefix
+        '--source-archive', sourceInfo.sourceArchive,
+        '--source-location-prefix', sourceInfo.sourceLocationPrefix
       );
     }
     args.push(resultsPath);
-    await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, "Interpreting query results");
+    await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
 
     let output: string;
     try {
       output = await fs.readFile(interpretedResultsPath, 'utf8');
     } catch (err) {
-      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`)
+      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
     }
     try {
       return JSON.parse(output) as sarif.Log;
     } catch (err) {
-      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`)
+      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
     }
   }
 
@@ -366,9 +595,9 @@ export class CodeQLCliServer implements Disposable {
     const sortDirectionStrings = sortDirections.map(direction => {
       switch (direction) {
         case SortDirection.asc:
-          return "asc";
+          return 'asc';
         case SortDirection.desc:
-          return "desc";
+          return 'desc';
         default:
           return assertNever(direction);
       }
@@ -376,14 +605,14 @@ export class CodeQLCliServer implements Disposable {
 
     await this.runCodeQlCliCommand(['bqrs', 'decode'],
       [
-        "--format=bqrs",
+        '--format=bqrs',
         `--result-set=${resultSet}`,
         `--output=${sortedResultsPath}`,
-        `--sort-key=${sortKeys.join(",")}`,
-        `--sort-direction=${sortDirectionStrings.join(",")}`,
+        `--sort-key=${sortKeys.join(',')}`,
+        `--sort-direction=${sortDirectionStrings.join(',')}`,
         resultsPath
       ],
-      "Sorting query results");
+      'Sorting query results');
   }
 
 
@@ -393,10 +622,9 @@ export class CodeQLCliServer implements Disposable {
    */
   resolveDatabase(databasePath: string): Promise<DbInfo> {
     return this.runJsonCodeQlCliCommand(['resolve', 'database'], [databasePath],
-      "Resolving database");
+      'Resolving database');
   }
 
-
   /**
    * Gets information necessary for upgrading a database.
    * @param dbScheme the path to the dbscheme of the database to be upgraded.
@@ -409,9 +637,83 @@ export class CodeQLCliServer implements Disposable {
     return this.runJsonCodeQlCliCommand<UpgradesInfo>(
       ['resolve', 'upgrades'],
       args,
-      "Resolving database upgrade scripts",
+      'Resolving database upgrade scripts',
     );
   }
+
+  /**
+   * Gets information about available qlpacks
+   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
+   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
+   *   the default CLI search path is used.
+   * @returns A dictionary mapping qlpack name to the directory it comes from
+   */
+  resolveQlpacks(additionalPacks: string[], searchPath?: string[]): Promise<QlpacksInfo> {
+    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    if (searchPath?.length) {
+      args.push('--search-path', path.join(...searchPath));
+    }
+
+    return this.runJsonCodeQlCliCommand<QlpacksInfo>(
+      ['resolve', 'qlpacks'],
+      args,
+      'Resolving qlpack information',
+    );
+  }
+
+  /**
+   * Gets information about queries in a query suite.
+   * @param suite The suite to resolve.
+   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
+   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
+   *   the default CLI search path is used.
+   * @returns A list of query files found.
+   */
+  resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
+    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    if (searchPath !== undefined) {
+      args.push('--search-path', path.join(...searchPath));
+    }
+    args.push(suite);
+    return this.runJsonCodeQlCliCommand<string[]>(
+      ['resolve', 'queries'],
+      args,
+      'Resolving queries',
+    );
+  }
+
+  async generateDil(qloFile: string, outFile: string): Promise<void> {
+    const extraArgs = (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0
+      ? ['--kind', 'dil', '-o', outFile, qloFile]
+      : ['-o', outFile, qloFile];
+    await this.runCodeQlCliCommand(
+      ['query', 'decompile'],
+      extraArgs,
+      'Generating DIL',
+    );
+  }
+
+  private async getVersion() {
+    if (!this._version) {
+      this._version = await this.refreshVersion();
+    }
+    return this._version;
+  }
+
+  private async refreshVersion() {
+    const distribution = await this.distributionProvider.getDistribution();
+    switch (distribution.kind) {
+      case FindDistributionResultKind.CompatibleDistribution:
+      // eslint-disable-next-line no-fallthrough
+      case FindDistributionResultKind.IncompatibleDistribution:
+        return distribution.version;
+
+      default:
+        // We should not get here because if no distributions are available, then
+        // the cli class is never instantiated.
+        throw new Error('No distribution found');
+    }
+  }
 }
 
 /**
@@ -443,7 +745,7 @@ export function spawnServer(
 
   // Start the server process.
   const base = codeqlPath;
-  const argsString = args.join(" ");
+  const argsString = args.join(' ');
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Starting ${name}` });
   }
@@ -469,7 +771,7 @@ export function spawnServer(
 
 /**
  * Runs a CodeQL CLI command without invoking the CLI server, returning the output as a string.
- * @param config The configuration containing the path to the CLI.
+ * @param codeQlPath The path to the CLI.
  * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
  * @param commandArgs The arguments to pass to the `codeql` command.
  * @param description Description of the action being run, to be shown in log and error messages.
@@ -477,20 +779,125 @@ export function spawnServer(
  * @param progressReporter Used to output progress messages, e.g. to the status bar.
  * @returns The contents of the command's stdout, if the command succeeded.
  */
-export async function runCodeQlCliCommand(codeQlPath: string, command: string[], commandArgs: string[], description: string, logger: Logger, progressReporter?: ProgressReporter): Promise<string> {
+export async function runCodeQlCliCommand(
+  codeQlPath: string,
+  command: string[],
+  commandArgs: string[],
+  description: string,
+  logger: Logger,
+  progressReporter?: ProgressReporter
+): Promise<string> {
   // Add logging arguments first, in case commandArgs contains positional parameters.
   const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
-  const argsString = args.join(" ");
+  const argsString = args.join(' ');
   try {
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
     }
     logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
-    const result = await util.promisify(child_process.execFile)(codeQlPath, args);
+    const result = await promisify(child_process.execFile)(codeQlPath, args);
     logger.log(result.stderr);
-    logger.log(`CLI command succeeded.`);
+    logger.log('CLI command succeeded.');
     return result.stdout;
   } catch (err) {
-    throw new Error(`${description} failed: ${err.stderr || err}`)
+    throw new Error(`${description} failed: ${err.stderr || err}`);
+  }
+}
+
+/**
+ * Buffer to hold state used when splitting a text stream into lines.
+ */
+class SplitBuffer {
+  private readonly decoder = new StringDecoder('utf8');
+  private readonly maxSeparatorLength: number;
+  private buffer = '';
+  private searchIndex = 0;
+
+  constructor(private readonly separators: readonly string[]) {
+    this.maxSeparatorLength = separators.map(s => s.length).reduce((a, b) => Math.max(a, b), 0);
+  }
+
+  /**
+   * Append new text data to the buffer.
+   * @param chunk The chunk of data to append.
+   */
+  public addChunk(chunk: Buffer): void {
+    this.buffer += this.decoder.write(chunk);
+  }
+
+  /**
+   * Signal that the end of the input stream has been reached.
+   */
+  public end(): void {
+    this.buffer += this.decoder.end();
+    this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
+  }
+
+  /**
+   * Extract the next full line from the buffer, if one is available.
+   * @returns The text of the next available full line (without the separator), or `undefined` if no
+   * line is available.
+   */
+  public getNextLine(): string | undefined {
+    while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
+      for (const separator of this.separators) {
+        if (this.buffer.startsWith(separator, this.searchIndex)) {
+          const line = this.buffer.substr(0, this.searchIndex);
+          this.buffer = this.buffer.substr(this.searchIndex + separator.length);
+          this.searchIndex = 0;
+          return line;
+        }
+      }
+      this.searchIndex++;
+    }
+
+    return undefined;
+  }
+}
+
+/**
+ * Splits a text stream into lines based on a list of valid line separators.
+ * @param stream The text stream to split. This stream will be fully consumed.
+ * @param separators The list of strings that act as line separators.
+ * @returns A sequence of lines (not including separators).
+ */
+async function* splitStreamAtSeparators(
+  stream: Readable, separators: string[]
+): AsyncGenerator<string, void, unknown> {
+
+  const buffer = new SplitBuffer(separators);
+  for await (const chunk of stream) {
+    buffer.addChunk(chunk);
+    let line: string | undefined;
+    do {
+      line = buffer.getNextLine();
+      if (line !== undefined) {
+        yield line;
+      }
+    } while (line !== undefined);
+  }
+  buffer.end();
+  let line: string | undefined;
+  do {
+    line = buffer.getNextLine();
+    if (line !== undefined) {
+      yield line;
+    }
+  } while (line !== undefined);
+}
+
+/**
+ *  Standard line endings for splitting human-readable text.
+ */
+const lineEndings = ['\r\n', '\r', '\n'];
+
+/**
+ * Log a text stream to a `Logger` interface.
+ * @param stream The stream to log.
+ * @param logger The logger that will consume the stream output.
+ */
+async function logStream(stream: Readable, logger: Logger): Promise<void> {
+  for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
+    logger.log(line);
   }
 }

extensions/ql-vscode/src/compare/compare-interface.ts

@@ -0,0 +1,277 @@
+import { DisposableObject } from '../vscode-utils/disposable-object';
+import {
+  WebviewPanel,
+  ExtensionContext,
+  window as Window,
+  ViewColumn,
+  Uri,
+} from 'vscode';
+import * as path from 'path';
+
+import { tmpDir } from '../run-queries';
+import { CompletedQuery } from '../query-results';
+import {
+  FromCompareViewMessage,
+  ToCompareViewMessage,
+  QueryCompareResult,
+} from '../pure/interface-types';
+import { Logger } from '../logging';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseManager } from '../databases';
+import { getHtmlForWebview, jumpToLocation } from '../interface-utils';
+import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../pure/bqrs-cli-types';
+import resultsDiff from './resultsDiff';
+
+interface ComparePair {
+  from: CompletedQuery;
+  to: CompletedQuery;
+}
+
+export class CompareInterfaceManager extends DisposableObject {
+  private comparePair: ComparePair | undefined;
+  private panel: WebviewPanel | undefined;
+  private panelLoaded = false;
+  private panelLoadedCallBacks: (() => void)[] = [];
+
+  constructor(
+    private ctx: ExtensionContext,
+    private databaseManager: DatabaseManager,
+    private cliServer: CodeQLCliServer,
+    private logger: Logger,
+    private showQueryResultsCallback: (
+      item: CompletedQuery
+    ) => Promise<void>
+  ) {
+    super();
+  }
+
+  async showResults(
+    from: CompletedQuery,
+    to: CompletedQuery,
+    selectedResultSetName?: string
+  ) {
+    this.comparePair = { from, to };
+    this.getPanel().reveal(undefined, true);
+
+    await this.waitForPanelLoaded();
+    const [
+      commonResultSetNames,
+      currentResultSetName,
+      fromResultSet,
+      toResultSet,
+    ] = await this.findCommonResultSetNames(
+      from,
+      to,
+      selectedResultSetName
+    );
+    if (currentResultSetName) {
+      let rows: QueryCompareResult | undefined;
+      let message: string | undefined;
+      try {
+        rows = this.compareResults(fromResultSet, toResultSet);
+      } catch (e) {
+        message = e.message;
+      }
+
+      await this.postMessage({
+        t: 'setComparisons',
+        stats: {
+          fromQuery: {
+            // since we split the description into several rows
+            // only run interpolation if the label is user-defined
+            // otherwise we will wind up with duplicated rows
+            name: from.options.label
+              ? from.interpolate(from.getLabel())
+              : from.queryName,
+            status: from.statusString,
+            time: from.time,
+          },
+          toQuery: {
+            name: to.options.label
+              ? to.interpolate(to.getLabel())
+              : to.queryName,
+            status: to.statusString,
+            time: to.time,
+          },
+        },
+        columns: fromResultSet.schema.columns,
+        commonResultSetNames,
+        currentResultSetName: currentResultSetName,
+        rows,
+        message,
+        datebaseUri: to.database.databaseUri,
+      });
+    }
+  }
+
+  getPanel(): WebviewPanel {
+    if (this.panel == undefined) {
+      const { ctx } = this;
+      const panel = (this.panel = Window.createWebviewPanel(
+        'compareView',
+        'Compare CodeQL Query Results',
+        { viewColumn: ViewColumn.Active, preserveFocus: true },
+        {
+          enableScripts: true,
+          enableFindWidget: true,
+          retainContextWhenHidden: true,
+          localResourceRoots: [
+            Uri.file(tmpDir.name),
+            Uri.file(path.join(this.ctx.extensionPath, 'out')),
+          ],
+        }
+      ));
+      this.panel.onDidDispose(
+        () => {
+          this.panel = undefined;
+          this.comparePair = undefined;
+        },
+        null,
+        ctx.subscriptions
+      );
+
+      const scriptPathOnDisk = Uri.file(
+        ctx.asAbsolutePath('out/compareView.js')
+      );
+
+      const stylesheetPathOnDisk = Uri.file(
+        ctx.asAbsolutePath('out/resultsView.css')
+      );
+
+      panel.webview.html = getHtmlForWebview(
+        panel.webview,
+        scriptPathOnDisk,
+        stylesheetPathOnDisk
+      );
+      panel.webview.onDidReceiveMessage(
+        async (e) => this.handleMsgFromView(e),
+        undefined,
+        ctx.subscriptions
+      );
+    }
+    return this.panel;
+  }
+
+  private waitForPanelLoaded(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.panelLoaded) {
+        resolve();
+      } else {
+        this.panelLoadedCallBacks.push(resolve);
+      }
+    });
+  }
+
+  private async handleMsgFromView(
+    msg: FromCompareViewMessage
+  ): Promise<void> {
+    switch (msg.t) {
+      case 'compareViewLoaded':
+        this.panelLoaded = true;
+        this.panelLoadedCallBacks.forEach((cb) => cb());
+        this.panelLoadedCallBacks = [];
+        break;
+
+      case 'changeCompare':
+        this.changeTable(msg.newResultSetName);
+        break;
+
+      case 'viewSourceFile':
+        await jumpToLocation(msg, this.databaseManager, this.logger);
+        break;
+
+      case 'openQuery':
+        await this.openQuery(msg.kind);
+        break;
+    }
+  }
+
+  private postMessage(msg: ToCompareViewMessage): Thenable<boolean> {
+    return this.getPanel().webview.postMessage(msg);
+  }
+
+  private async findCommonResultSetNames(
+    from: CompletedQuery,
+    to: CompletedQuery,
+    selectedResultSetName: string | undefined
+  ): Promise<[string[], string, RawResultSet, RawResultSet]> {
+    const fromSchemas = await this.cliServer.bqrsInfo(
+      from.query.resultsPaths.resultsPath
+    );
+    const toSchemas = await this.cliServer.bqrsInfo(
+      to.query.resultsPaths.resultsPath
+    );
+    const fromSchemaNames = fromSchemas['result-sets'].map(
+      (schema) => schema.name
+    );
+    const toSchemaNames = toSchemas['result-sets'].map(
+      (schema) => schema.name
+    );
+    const commonResultSetNames = fromSchemaNames.filter((name) =>
+      toSchemaNames.includes(name)
+    );
+    const currentResultSetName =
+      selectedResultSetName || commonResultSetNames[0];
+    const fromResultSet = await this.getResultSet(
+      fromSchemas,
+      currentResultSetName,
+      from.query.resultsPaths.resultsPath
+    );
+    const toResultSet = await this.getResultSet(
+      toSchemas,
+      currentResultSetName,
+      to.query.resultsPaths.resultsPath
+    );
+    return [
+      commonResultSetNames,
+      currentResultSetName,
+      fromResultSet,
+      toResultSet,
+    ];
+  }
+
+  private async changeTable(newResultSetName: string) {
+    if (!this.comparePair?.from || !this.comparePair.to) {
+      return;
+    }
+    await this.showResults(
+      this.comparePair.from,
+      this.comparePair.to,
+      newResultSetName
+    );
+  }
+
+  private async getResultSet(
+    bqrsInfo: BQRSInfo,
+    resultSetName: string,
+    resultsPath: string
+  ): Promise<RawResultSet> {
+    const schema = bqrsInfo['result-sets'].find(
+      (schema) => schema.name === resultSetName
+    );
+    if (!schema) {
+      throw new Error(`Schema ${resultSetName} not found.`);
+    }
+    const chunk = await this.cliServer.bqrsDecode(
+      resultsPath,
+      resultSetName
+    );
+    return transformBqrsResultSet(schema, chunk);
+  }
+
+  private compareResults(
+    fromResults: RawResultSet,
+    toResults: RawResultSet
+  ): QueryCompareResult {
+    // Only compare columns that have the same name
+    return resultsDiff(fromResults, toResults);
+  }
+
+  private openQuery(kind: 'from' | 'to') {
+    const toOpen =
+      kind === 'from' ? this.comparePair?.from : this.comparePair?.to;
+    if (toOpen) {
+      this.showQueryResultsCallback(toOpen);
+    }
+  }
+}

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -0,0 +1,58 @@
+import { RawResultSet } from '../pure/bqrs-cli-types';
+import { QueryCompareResult } from '../pure/interface-types';
+
+/**
+ * Compare the rows of two queries. Use deep equality to determine if
+ * rows have been added or removed across two invocations of a query.
+ *
+ * Assumptions:
+ *
+ * 1. Queries have the same sort order
+ * 2. Queries have same number and order of columns
+ * 3. Rows are not changed or re-ordered, they are only added or removed
+ *
+ * @param fromResults the source query
+ * @param toResults the target query
+ *
+ * @throws Error when:
+ *  1. number of columns do not match
+ *  2. If either query is empty
+ *  3. If the queries are 100% disjoint
+ */
+export default function resultsDiff(
+  fromResults: RawResultSet,
+  toResults: RawResultSet
+): QueryCompareResult {
+
+  if (fromResults.schema.columns.length !== toResults.schema.columns.length) {
+    throw new Error('CodeQL Compare: Columns do not match.');
+  }
+
+  if (!fromResults.rows.length) {
+    throw new Error('CodeQL Compare: Source query has no results.');
+  }
+
+  if (!toResults.rows.length) {
+    throw new Error('CodeQL Compare: Target query has no results.');
+  }
+
+  const results = {
+    from: arrayDiff(fromResults.rows, toResults.rows),
+    to: arrayDiff(toResults.rows, fromResults.rows),
+  };
+
+  if (
+    fromResults.rows.length === results.from.length &&
+    toResults.rows.length === results.to.length
+  ) {
+    throw new Error('CodeQL Compare: No overlap between the selected queries.');
+  }
+
+  return results;
+}
+
+function arrayDiff<T>(source: readonly T[], toRemove: readonly T[]): T[] {
+  // Stringify the object so that we can compare hashes in the set
+  const rest = new Set(toRemove.map((item) => JSON.stringify(item)));
+  return source.filter((element) => !rest.has(JSON.stringify(element)));
+}

extensions/ql-vscode/src/compare/view/.eslintrc.js

@@ -0,0 +1,13 @@
+module.exports = {
+  env: {
+    browser: true
+  },
+  extends: [
+    "plugin:react/recommended"
+  ],
+  settings: {
+    react: {
+      version: 'detect'
+    }
+  }
+}

extensions/ql-vscode/src/compare/view/Compare.tsx

@@ -0,0 +1,82 @@
+import * as React from 'react';
+import { useState, useEffect } from 'react';
+import * as Rdom from 'react-dom';
+
+import {
+  ToCompareViewMessage,
+  SetComparisonsMessage,
+} from '../../pure/interface-types';
+import CompareSelector from './CompareSelector';
+import { vscode } from '../../view/vscode-api';
+import CompareTable from './CompareTable';
+
+const emptyComparison: SetComparisonsMessage = {
+  t: 'setComparisons',
+  stats: {},
+  rows: undefined,
+  columns: [],
+  commonResultSetNames: [],
+  currentResultSetName: '',
+  datebaseUri: '',
+  message: 'Empty comparison'
+};
+
+export function Compare(_: {}): JSX.Element {
+  const [comparison, setComparison] = useState<SetComparisonsMessage>(
+    emptyComparison
+  );
+
+  const message = comparison.message || 'Empty comparison';
+  const hasRows = comparison.rows && (comparison.rows.to.length || comparison.rows.from.length);
+
+  useEffect(() => {
+    window.addEventListener('message', (evt: MessageEvent) => {
+      if (evt.origin === window.origin) {
+        const msg: ToCompareViewMessage = evt.data;
+        switch (msg.t) {
+          case 'setComparisons':
+            setComparison(msg);
+        }
+      } else {
+        console.error(`Invalid event origin ${evt.origin}`);
+      }
+    });
+  });
+  if (!comparison) {
+    return <div>Waiting for results to load.</div>;
+  }
+
+  try {
+    return (
+      <>
+        <div className="vscode-codeql__compare-header">
+          <div className="vscode-codeql__compare-header-item">
+            Table to compare:
+          </div>
+          <CompareSelector
+            availableResultSets={comparison.commonResultSetNames}
+            currentResultSetName={comparison.currentResultSetName}
+            updateResultSet={(newResultSetName: string) =>
+              vscode.postMessage({ t: 'changeCompare', newResultSetName })
+            }
+          />
+        </div>
+        {hasRows ? (
+          <CompareTable comparison={comparison}></CompareTable>
+        ) : (
+            <div className="vscode-codeql__compare-message">{message}</div>
+          )}
+      </>
+    );
+  } catch (err) {
+    console.error(err);
+    return <div>Error!</div>;
+  }
+}
+
+Rdom.render(
+  <Compare />,
+  document.getElementById('root'),
+  // Post a message to the extension when fully loaded.
+  () => vscode.postMessage({ t: 'compareViewLoaded' })
+);

extensions/ql-vscode/src/compare/view/CompareSelector.tsx

@@ -0,0 +1,22 @@
+import * as React from 'react';
+
+interface Props {
+  availableResultSets: string[];
+  currentResultSetName: string;
+  updateResultSet: (newResultSet: string) => void;
+}
+
+export default function CompareSelector(props: Props) {
+  return (
+    <select
+      value={props.currentResultSetName}
+      onChange={(e) => props.updateResultSet(e.target.value)}
+    >
+      {props.availableResultSets.map((resultSet) => (
+        <option key={resultSet} value={resultSet}>
+          {resultSet}
+        </option>
+      ))}
+    </select>
+  );
+}

extensions/ql-vscode/src/compare/view/CompareTable.tsx

@@ -0,0 +1,96 @@
+import * as React from 'react';
+
+import { SetComparisonsMessage } from '../../pure/interface-types';
+import RawTableHeader from '../../view/RawTableHeader';
+import { className } from '../../view/result-table-utils';
+import { ResultRow } from '../../pure/bqrs-cli-types';
+import RawTableRow from '../../view/RawTableRow';
+import { vscode } from '../../view/vscode-api';
+
+interface Props {
+  comparison: SetComparisonsMessage;
+}
+
+export default function CompareTable(props: Props) {
+  const comparison = props.comparison;
+  const rows = props.comparison.rows!;
+
+  async function openQuery(kind: 'from' | 'to') {
+    vscode.postMessage({
+      t: 'openQuery',
+      kind,
+    });
+  }
+
+  function createRows(rows: ResultRow[], databaseUri: string) {
+    return (
+      <tbody>
+        {rows.map((row, rowIndex) => (
+          <RawTableRow
+            key={rowIndex}
+            rowIndex={rowIndex}
+            row={row}
+            databaseUri={databaseUri}
+          />
+        ))}
+      </tbody>
+    );
+  }
+
+  return (
+    <table className='vscode-codeql__compare-body'>
+      <thead>
+        <tr>
+          <td>
+            <a
+              onClick={() => openQuery('from')}
+              className='vscode-codeql__compare-open'
+            >
+              {comparison.stats.fromQuery?.name}
+            </a>
+          </td>
+          <td>
+            <a
+              onClick={() => openQuery('to')}
+              className='vscode-codeql__compare-open'
+            >
+              {comparison.stats.toQuery?.name}
+            </a>
+          </td>
+        </tr>
+        <tr>
+          <td>{comparison.stats.fromQuery?.time}</td>
+          <td>{comparison.stats.toQuery?.time}</td>
+        </tr>
+        <tr>
+          <th>{rows.from.length} rows removed</th>
+          <th>{rows.to.length} rows added</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>
+            <table className={className}>
+              <RawTableHeader
+                columns={comparison.columns}
+                schemaName={comparison.currentResultSetName}
+                preventSort={true}
+              />
+              {createRows(rows.from, comparison.datebaseUri)}
+            </table>
+          </td>
+          <td>
+            <table className={className}>
+              <RawTableHeader
+                columns={comparison.columns}
+                schemaName={comparison.currentResultSetName}
+                preventSort={true}
+              />
+              {createRows(rows.to, comparison.datebaseUri)}
+            </table>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  );
+}

extensions/ql-vscode/src/compare/view/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "module": "esnext",
+    "moduleResolution": "node",
+    "target": "es6",
+    "outDir": "out",
+    "lib": [
+      "es6",
+      "dom"
+    ],
+    "jsx": "react",
+    "sourceMap": true,
+    "rootDir": "..",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "experimentalDecorators": true
+  },
+  "exclude": [
+    "node_modules"
+  ]
+}

extensions/ql-vscode/src/config.ts

@@ -1,5 +1,5 @@
-import { DisposableObject } from 'semmle-vscode-utils';
-import { workspace, Event, EventEmitter, ConfigurationChangeEvent } from 'vscode';
+import { DisposableObject } from './vscode-utils/disposable-object';
+import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
 
@@ -27,6 +27,14 @@ class Setting {
     }
     return workspace.getConfiguration(this.parent.qualifiedName).get<T>(this.name)!;
   }
+
+  updateValue<T>(value: T, target: ConfigurationTarget): Thenable<void> {
+    if (this.parent === undefined) {
+      throw new Error('Cannot update the value of a root setting.');
+    }
+    return workspace.getConfiguration(this.parent.qualifiedName).update(this.name, value, target);
+  }
+
 }
 
 const ROOT_SETTING = new Setting('codeQL');
@@ -49,7 +57,7 @@ export interface DistributionConfig {
   personalAccessToken?: string;
   ownerName?: string;
   repositoryName?: string;
-  onDidChangeDistributionConfiguration?: Event<void>;
+  onDidChangeConfiguration?: Event<void>;
 }
 
 // Query server configuration
@@ -59,27 +67,40 @@ const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
+const RUNNING_TESTS_SETTING = new Setting('runningTests', ROOT_SETTING);
+
+export const NUMBER_OF_TEST_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_TESTS_SETTING);
+export const MAX_QUERIES = new Setting('maxQueries', RUNNING_QUERIES_SETTING);
+export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
 const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
 
 export interface QueryServerConfig {
-  codeQlPath: string,
-  debug: boolean,
-  numThreads: number,
-  queryMemoryMb?: number,
-  timeoutSecs: number,
-  onDidChangeQueryServerConfiguration?: Event<void>;
+  codeQlPath: string;
+  debug: boolean;
+  numThreads: number;
+  queryMemoryMb?: number;
+  timeoutSecs: number;
+  onDidChangeConfiguration?: Event<void>;
 }
 
 /** When these settings change, the query history should be refreshed. */
 const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING];
 
 export interface QueryHistoryConfig {
-  format: string,
-  onDidChangeQueryHistoryConfiguration: Event<void>;
+  format: string;
+  onDidChangeConfiguration: Event<void>;
 }
 
+const CLI_SETTINGS = [NUMBER_OF_TEST_THREADS_SETTING];
+
+export interface CliConfig {
+  numberTestThreads: number;
+  onDidChangeConfiguration?: Event<void>;
+}
+
+
 abstract class ConfigListener extends DisposableObject {
   protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());
 
@@ -107,11 +128,15 @@ abstract class ConfigListener extends DisposableObject {
   private updateConfiguration(): void {
     this._onDidChangeConfiguration.fire();
   }
+
+  public get onDidChangeConfiguration(): Event<void> {
+    return this._onDidChangeConfiguration.event;
+  }
 }
 
 export class DistributionConfigListener extends ConfigListener implements DistributionConfig {
   public get customCodeQlPath(): string | undefined {
-    return CUSTOM_CODEQL_PATH_SETTING.getValue() ? CUSTOM_CODEQL_PATH_SETTING.getValue() : undefined;
+    return CUSTOM_CODEQL_PATH_SETTING.getValue() || undefined;
   }
 
   public get includePrerelease(): boolean {
@@ -119,11 +144,7 @@ export class DistributionConfigListener extends ConfigListener implements Distri
   }
 
   public get personalAccessToken(): string | undefined {
-    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() ? PERSONAL_ACCESS_TOKEN_SETTING.getValue() : undefined;
-  }
-
-  public get onDidChangeDistributionConfiguration(): Event<void> {
-    return this._onDidChangeConfiguration.event;
+    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
@@ -132,7 +153,7 @@ export class DistributionConfigListener extends ConfigListener implements Distri
 }
 
 export class QueryServerConfigListener extends ConfigListener implements QueryServerConfig {
-  private constructor(private _codeQlPath: string) {
+  public constructor(private _codeQlPath = '') {
     super();
   }
 
@@ -178,10 +199,6 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     return DEBUG_SETTING.getValue<boolean>();
   }
 
-  public get onDidChangeQueryServerConfiguration(): Event<void> {
-    return this._onDidChangeConfiguration.event;
-  }
-
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(QUERY_SERVER_RESTARTING_SETTINGS, e);
   }
@@ -192,11 +209,27 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
     this.handleDidChangeConfigurationForRelevantSettings(QUERY_HISTORY_SETTINGS, e);
   }
 
-  public get onDidChangeQueryHistoryConfiguration(): Event<void> {
-    return this._onDidChangeConfiguration.event;
-  }
-
   public get format(): string {
     return QUERY_HISTORY_FORMAT_SETTING.getValue<string>();
   }
 }
+
+export class CliConfigListener extends ConfigListener implements CliConfig {
+
+  public get numberTestThreads(): number {
+    return NUMBER_OF_TEST_THREADS_SETTING.getValue();
+  }
+
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
+  }
+}
+
+// Enable experimental features
+
+/**
+ * Any settings below are deliberately not in package.json so that
+ * they do not appear in the settings ui in vscode itself. If users
+ * want to enable experimental features, they can add them directly in
+ * their vscode settings json file.
+ */

extensions/ql-vscode/src/contextual/astBuilder.ts

@@ -0,0 +1,137 @@
+import { QueryWithResults } from '../run-queries';
+import { CodeQLCliServer } from '../cli';
+import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
+import { DatabaseItem } from '../databases';
+import { ChildAstItem, AstItem } from '../astViewer';
+import fileRangeFromURI from './fileRangeFromURI';
+
+/**
+ * A class that wraps a tree of QL results from a query that
+ * has an @kind of graph
+ */
+export default class AstBuilder {
+
+  private roots: AstItem[] | undefined;
+  private bqrsPath: string;
+  constructor(
+    queryResults: QueryWithResults,
+    private cli: CodeQLCliServer,
+    public db: DatabaseItem,
+    public fileName: string
+  ) {
+    this.bqrsPath = queryResults.query.resultsPaths.resultsPath;
+  }
+
+  async getRoots(): Promise<AstItem[]> {
+    if (!this.roots) {
+      this.roots = await this.parseRoots();
+    }
+    return this.roots;
+  }
+
+  private async parseRoots(): Promise<AstItem[]> {
+    const options = { entities: ['id', 'url', 'string'] };
+    const [nodeTuples, edgeTuples, graphProperties] = await Promise.all([
+      await this.cli.bqrsDecode(this.bqrsPath, 'nodes', options),
+      await this.cli.bqrsDecode(this.bqrsPath, 'edges', options),
+      await this.cli.bqrsDecode(this.bqrsPath, 'graphProperties', options),
+    ]);
+
+    if (!this.isValidGraph(graphProperties)) {
+      throw new Error('AST is invalid');
+    }
+
+    const idToItem = new Map<BqrsId, AstItem>();
+    const parentToChildren = new Map<BqrsId, BqrsId[]>();
+    const childToParent = new Map<BqrsId, BqrsId>();
+    const astOrder = new Map<BqrsId, number>();
+    const roots = [];
+
+    // Build up the parent-child relationships
+    edgeTuples.tuples.forEach(tuple => {
+      const [source, target, tupleType, orderValue] = tuple as [EntityValue, EntityValue, string, string];
+      const sourceId = source.id!;
+      const targetId = target.id!;
+
+      switch (tupleType) {
+        case 'semmle.order':
+          astOrder.set(targetId, Number(orderValue));
+          break;
+
+        case 'semmle.label': {
+          childToParent.set(targetId, sourceId);
+          let children = parentToChildren.get(sourceId);
+          if (!children) {
+            parentToChildren.set(sourceId, children = []);
+          }
+          children.push(targetId);
+          break;
+        }
+
+        default:
+        // ignore other tupleTypes since they are not needed by the ast viewer
+      }
+    });
+
+    // populate parents and children
+    nodeTuples.tuples.forEach(tuple => {
+      const [entity, tupleType, orderValue] = tuple as [EntityValue, string, string];
+      const id = entity.id!;
+
+      switch (tupleType) {
+        case 'semmle.order':
+          astOrder.set(id, Number(orderValue));
+          break;
+
+        case 'semmle.label': {
+          const item = {
+            id,
+            label: entity.label,
+            location: entity.url,
+            fileLocation: fileRangeFromURI(entity.url, this.db),
+            children: [] as ChildAstItem[],
+            order: Number.MAX_SAFE_INTEGER
+          };
+
+          idToItem.set(id, item);
+          const parent = idToItem.get(childToParent.has(id) ? childToParent.get(id)! : -1);
+
+          if (parent) {
+            const astItem = item as ChildAstItem;
+            astItem.parent = parent;
+            parent.children.push(astItem);
+          }
+          const children = parentToChildren.has(id) ? parentToChildren.get(id)! : [];
+          children.forEach(childId => {
+            const child = idToItem.get(childId) as ChildAstItem | undefined;
+            if (child) {
+              child.parent = item;
+              item.children.push(child);
+            }
+          });
+          break;
+        }
+
+        default:
+        // ignore other tupleTypes since they are not needed by the ast viewer
+      }
+    });
+
+    // find the roots and add the order
+    for (const [, item] of idToItem) {
+      item.order = astOrder.has(item.id)
+        ? astOrder.get(item.id)!
+        : Number.MAX_SAFE_INTEGER;
+
+      if (!('parent' in item)) {
+        roots.push(item);
+      }
+    }
+    return roots;
+  }
+
+  private isValidGraph(graphProperties: DecodedBqrsChunk) {
+    const tuple = graphProperties?.tuples?.find(t => t[0] === 'semmle.graphKind');
+    return tuple?.[1] === 'tree';
+  }
+}

extensions/ql-vscode/src/contextual/fileRangeFromURI.ts

@@ -0,0 +1,31 @@
+import * as vscode from 'vscode';
+
+import { UrlValue, LineColumnLocation } from '../pure/bqrs-cli-types';
+import { isEmptyPath } from '../pure/bqrs-utils';
+import { DatabaseItem } from '../databases';
+
+
+export default function fileRangeFromURI(uri: UrlValue | undefined, db: DatabaseItem): vscode.Location | undefined {
+  if (!uri || typeof uri === 'string') {
+    return undefined;
+  } else if ('startOffset' in uri) {
+    return undefined;
+  } else {
+    const loc = uri as LineColumnLocation;
+    if (isEmptyPath(loc.uri)) {
+      return undefined;
+    }
+    const range = new vscode.Range(Math.max(0, (loc.startLine || 0) - 1),
+      Math.max(0, (loc.startColumn || 0) - 1),
+      Math.max(0, (loc.endLine || 0) - 1),
+      Math.max(0, (loc.endColumn || 0)));
+    try {
+      if (uri.uri.startsWith('file:')) {
+        return new vscode.Location(db.resolveSourceFile(uri.uri), range);
+      }
+      return undefined;
+    } catch (e) {
+      return undefined;
+    }
+  }
+}

extensions/ql-vscode/src/contextual/keyType.ts

@@ -0,0 +1,37 @@
+export enum KeyType {
+  DefinitionQuery = 'DefinitionQuery',
+  ReferenceQuery = 'ReferenceQuery',
+  PrintAstQuery = 'PrintAstQuery',
+}
+
+export function tagOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery:
+      return 'ide-contextual-queries/local-definitions';
+    case KeyType.ReferenceQuery:
+      return 'ide-contextual-queries/local-references';
+    case KeyType.PrintAstQuery:
+      return 'ide-contextual-queries/print-ast';
+  }
+}
+
+export function nameOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery:
+      return 'definitions';
+    case KeyType.ReferenceQuery:
+      return 'references';
+    case KeyType.PrintAstQuery:
+      return 'print AST';
+  }
+}
+
+export function kindOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery:
+    case KeyType.ReferenceQuery:
+      return 'definitions';
+    case KeyType.PrintAstQuery:
+      return 'graph';
+  }
+}

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -0,0 +1,127 @@
+import * as vscode from 'vscode';
+
+import { decodeSourceArchiveUri, encodeArchiveBasePath } from '../archive-filesystem-provider';
+import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from '../pure/bqrs-cli-types';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseManager, DatabaseItem } from '../databases';
+import fileRangeFromURI from './fileRangeFromURI';
+import * as messages from '../pure/messages';
+import { QueryServerClient } from '../queryserver-client';
+import { QueryWithResults, compileAndRunQueryAgainstDatabase } from '../run-queries';
+import { ProgressCallback } from '../helpers';
+import { KeyType } from './keyType';
+import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+
+const SELECT_QUERY_NAME = '#select';
+export const TEMPLATE_NAME = 'selectedSourceFile';
+
+export interface FullLocationLink extends vscode.LocationLink {
+  originUri: vscode.Uri;
+}
+
+/**
+ * This function executes a contextual query inside a given database, filters, and converts
+ * the results into source locations. This function is the workhorse for all search-based
+ * contextual queries like find references and find definitions.
+ *
+ * @param cli The cli server
+ * @param qs The query server client
+ * @param dbm The database manager
+ * @param uriString The selected source file and location
+ * @param keyType The contextual query type to run
+ * @param progress A progress callback
+ * @param token A CancellationToken
+ * @param filter A function that will filter extraneous results
+ */
+export async function getLocationsForUriString(
+  cli: CodeQLCliServer,
+  qs: QueryServerClient,
+  dbm: DatabaseManager,
+  uriString: string,
+  keyType: KeyType,
+  progress: ProgressCallback,
+  token: vscode.CancellationToken,
+  filter: (src: string, dest: string) => boolean
+): Promise<FullLocationLink[]> {
+  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString, true));
+  const sourceArchiveUri = encodeArchiveBasePath(uri.sourceArchiveZipPath);
+
+  const db = dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+  if (!db) {
+    return [];
+  }
+
+  const qlpack = await qlpackOfDatabase(cli, db);
+  if (qlpack === undefined) {
+    throw new Error('Can\'t infer qlpack from database source archive');
+  }
+  const templates = createTemplates(uri.pathWithinSourceArchive);
+
+  const links: FullLocationLink[] = [];
+  for (const query of await resolveQueries(cli, qlpack, keyType)) {
+    const results = await compileAndRunQueryAgainstDatabase(
+      cli,
+      qs,
+      db,
+      false,
+      vscode.Uri.file(query),
+      progress,
+      token,
+      templates
+    );
+
+    if (results.result.resultType == messages.QueryResultType.SUCCESS) {
+      links.push(...await getLinksFromResults(results, cli, db, filter));
+    }
+  }
+  return links;
+}
+
+async function getLinksFromResults(
+  results: QueryWithResults,
+  cli: CodeQLCliServer,
+  db: DatabaseItem,
+  filter: (srcFile: string, destFile: string) => boolean
+): Promise<FullLocationLink[]> {
+  const localLinks: FullLocationLink[] = [];
+  const bqrsPath = results.query.resultsPaths.resultsPath;
+  const info = await cli.bqrsInfo(bqrsPath);
+  const selectInfo = getResultSetSchema(SELECT_QUERY_NAME, info);
+  if (isValidSelect(selectInfo)) {
+    // TODO: Page this
+    const allTuples = await cli.bqrsDecode(bqrsPath, SELECT_QUERY_NAME);
+    for (const tuple of allTuples.tuples) {
+      const [src, dest] = tuple as [EntityValue, EntityValue];
+      const srcFile = src.url && fileRangeFromURI(src.url, db);
+      const destFile = dest.url && fileRangeFromURI(dest.url, db);
+      if (srcFile && destFile && filter(srcFile.uri.toString(), destFile.uri.toString())) {
+        localLinks.push({
+          targetRange: destFile.range,
+          targetUri: destFile.uri,
+          originSelectionRange: srcFile.range,
+          originUri: srcFile.uri
+        });
+      }
+    }
+  }
+  return localLinks;
+}
+
+function createTemplates(path: string): messages.TemplateDefinitions {
+  return {
+    [TEMPLATE_NAME]: {
+      values: {
+        tuples: [[{
+          stringValue: path
+        }]]
+      }
+    }
+  };
+}
+
+function isValidSelect(selectInfo: ResultSetSchema | undefined) {
+  return selectInfo && selectInfo.columns.length == 3
+    && selectInfo.columns[0].kind == ColumnKindCode.ENTITY
+    && selectInfo.columns[1].kind == ColumnKindCode.ENTITY
+    && selectInfo.columns[2].kind == ColumnKindCode.STRING;
+}

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -0,0 +1,47 @@
+import * as fs from 'fs-extra';
+import * as yaml from 'js-yaml';
+import * as tmp from 'tmp-promise';
+
+import * as helpers from '../helpers';
+import {
+  KeyType,
+  kindOfKeyType,
+  nameOfKeyType,
+  tagOfKeyType
+} from './keyType';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseItem } from '../databases';
+
+export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string | undefined> {
+  if (db.contents === undefined)
+    return undefined;
+  const datasetPath = db.contents.datasetUri.fsPath;
+  const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath);
+  return qlpack;
+}
+
+
+export async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
+  const suiteFile = (await tmp.file({
+    postfix: '.qls'
+  })).path;
+  const suiteYaml = {
+    qlpack,
+    include: {
+      kind: kindOfKeyType(keyType),
+      'tags contain': tagOfKeyType(keyType)
+    }
+  };
+  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
+
+  const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
+  if (queries.length === 0) {
+    helpers.showAndLogErrorMessage(
+      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
+      Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
+      for this language.`
+    );
+    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
+  }
+  return queries;
+}

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -0,0 +1,192 @@
+import * as vscode from 'vscode';
+
+import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseManager } from '../databases';
+import { CachedOperation, ProgressCallback, withProgress } from '../helpers';
+import * as messages from '../pure/messages';
+import { QueryServerClient } from '../queryserver-client';
+import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';
+import AstBuilder from './astBuilder';
+import {
+  KeyType,
+} from './keyType';
+import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
+import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+
+/**
+ * Run templated CodeQL queries to find definitions and references in
+ * source-language files. We may eventually want to find a way to
+ * generalize this to other custom queries, e.g. showing dataflow to
+ * or from a selected identifier.
+ */
+
+export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
+  private cache: CachedOperation<vscode.LocationLink[]>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+  }
+
+  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+    const fileLinks = await this.cache.get(document.uri.toString());
+    const locLinks: vscode.LocationLink[] = [];
+    for (const link of fileLinks) {
+      if (link.originSelectionRange!.contains(position)) {
+        locLinks.push(link);
+      }
+    }
+    return locLinks;
+  }
+
+  private async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+    return withProgress({
+      location: vscode.ProgressLocation.Notification,
+      cancellable: true,
+      title: 'Finding definitions'
+    }, async (progress, token) => {
+      return getLocationsForUriString(
+        this.cli,
+        this.qs,
+        this.dbm,
+        uriString,
+        KeyType.DefinitionQuery,
+        progress,
+        token,
+        (src, _dest) => src === uriString
+      );
+    });
+  }
+}
+
+export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+  private cache: CachedOperation<FullLocationLink[]>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<FullLocationLink[]>(this.getReferences.bind(this));
+  }
+
+  async provideReferences(
+    document: vscode.TextDocument,
+    position: vscode.Position,
+    _context: vscode.ReferenceContext,
+    _token: vscode.CancellationToken
+  ): Promise<vscode.Location[]> {
+    const fileLinks = await this.cache.get(document.uri.toString());
+    const locLinks: vscode.Location[] = [];
+    for (const link of fileLinks) {
+      if (link.targetRange!.contains(position)) {
+        locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
+      }
+    }
+    return locLinks;
+  }
+
+  private async getReferences(uriString: string): Promise<FullLocationLink[]> {
+    return withProgress({
+      location: vscode.ProgressLocation.Notification,
+      cancellable: true,
+      title: 'Finding references'
+    }, async (progress, token) => {
+      return getLocationsForUriString(
+        this.cli,
+        this.qs,
+        this.dbm,
+        uriString,
+        KeyType.DefinitionQuery,
+        progress,
+        token,
+        (src, _dest) => src === uriString
+      );
+    });
+  }
+}
+
+export class TemplatePrintAstProvider {
+  private cache: CachedOperation<QueryWithResults | undefined>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+
+    // Note: progress and token are only used if a cached value is not available
+    private progress: ProgressCallback,
+    private token: vscode.CancellationToken
+  ) {
+    this.cache = new CachedOperation<QueryWithResults | undefined>(this.getAst.bind(this));
+  }
+
+  async provideAst(document?: vscode.TextDocument): Promise<AstBuilder | undefined> {
+    if (!document) {
+      return;
+    }
+    const queryResults = await this.cache.get(document.uri.toString());
+    if (!queryResults) {
+      return;
+    }
+
+    return new AstBuilder(
+      queryResults, this.cli,
+      this.dbm.findDatabaseItem(vscode.Uri.parse(queryResults.database.databaseUri!, true))!,
+      document.fileName
+    );
+  }
+
+  private async getAst(uriString: string): Promise<QueryWithResults> {
+    const uri = vscode.Uri.parse(uriString, true);
+    if (uri.scheme !== zipArchiveScheme) {
+      throw new Error('AST Viewing is only available for databases with zipped source archives.');
+    }
+
+    const zippedArchive = decodeSourceArchiveUri(uri);
+    const sourceArchiveUri = encodeArchiveBasePath(zippedArchive.sourceArchiveZipPath);
+    const db = this.dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+
+    if (!db) {
+      throw new Error('Can\'t infer database from the provided source.');
+    }
+
+    const qlpack = await qlpackOfDatabase(this.cli, db);
+    if (!qlpack) {
+      throw new Error('Can\'t infer qlpack from database source archive');
+    }
+    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintAstQuery);
+    if (queries.length > 1) {
+      throw new Error('Found multiple Print AST queries. Can\'t continue');
+    }
+    if (queries.length === 0) {
+      throw new Error('Did not find any Print AST queries. Can\'t continue');
+    }
+
+    const query = queries[0];
+    const templates: messages.TemplateDefinitions = {
+      [TEMPLATE_NAME]: {
+        values: {
+          tuples: [[{
+            stringValue: zippedArchive.pathWithinSourceArchive
+          }]]
+        }
+      }
+    };
+
+    return await compileAndRunQueryAgainstDatabase(
+      this.cli,
+      this.qs,
+      db,
+      false,
+      vscode.Uri.file(query),
+      this.progress,
+      this.token,
+      templates
+    );
+  }
+}

extensions/ql-vscode/src/databaseFetcher.ts

@@ -0,0 +1,430 @@
+import fetch, { Response } from 'node-fetch';
+import * as unzipper from 'unzipper';
+import { zip } from 'zip-a-folder';
+import {
+  Uri,
+  CancellationToken,
+  commands,
+  window,
+} from 'vscode';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { DatabaseManager, DatabaseItem } from './databases';
+import {
+  ProgressCallback,
+  showAndLogInformationMessage,
+} from './helpers';
+import { logger } from './logging';
+
+/**
+ * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
+ *
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function promptImportInternetDatabase(
+  databasesManager: DatabaseManager,
+  storagePath: string,
+  progress: ProgressCallback,
+  _: CancellationToken,
+): Promise<DatabaseItem | undefined> {
+  const databaseUrl = await window.showInputBox({
+    prompt: 'Enter URL of zipfile of database to download',
+  });
+  if (!databaseUrl) {
+    return;
+  }
+
+  validateHttpsUrl(databaseUrl);
+
+  const item = await databaseArchiveFetcher(
+    databaseUrl,
+    databasesManager,
+    storagePath,
+    progress
+  );
+
+  if (item) {
+    commands.executeCommand('codeQLDatabases.focus');
+    showAndLogInformationMessage('Database downloaded and imported successfully.');
+  }
+  return item;
+
+}
+
+/**
+ * Prompts a user to fetch a database from lgtm.
+ * User enters a project url and then the user is asked which language
+ * to download (if there is more than one)
+ *
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function promptImportLgtmDatabase(
+  databasesManager: DatabaseManager,
+  storagePath: string,
+  progress: ProgressCallback,
+  _: CancellationToken
+): Promise<DatabaseItem | undefined> {
+  const lgtmUrl = await window.showInputBox({
+    prompt:
+      'Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)',
+  });
+  if (!lgtmUrl) {
+    return;
+  }
+
+  if (looksLikeLgtmUrl(lgtmUrl)) {
+    const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
+    if (databaseUrl) {
+      const item = await databaseArchiveFetcher(
+        databaseUrl,
+        databasesManager,
+        storagePath,
+        progress
+      );
+      if (item) {
+        commands.executeCommand('codeQLDatabases.focus');
+        showAndLogInformationMessage('Database downloaded and imported successfully.');
+      }
+      return item;
+    }
+  } else {
+    throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+  }
+  return;
+}
+
+/**
+ * Imports a database from a local archive.
+ *
+ * @param databaseUrl the file url of the archive to import
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function importArchiveDatabase(
+  databaseUrl: string,
+  databasesManager: DatabaseManager,
+  storagePath: string,
+  progress: ProgressCallback,
+  _: CancellationToken,
+): Promise<DatabaseItem | undefined> {
+  try {
+    const item = await databaseArchiveFetcher(
+      databaseUrl,
+      databasesManager,
+      storagePath,
+      progress
+    );
+    if (item) {
+      commands.executeCommand('codeQLDatabases.focus');
+      showAndLogInformationMessage('Database unzipped and imported successfully.');
+    }
+    return item;
+  } catch (e) {
+    if (e.message.includes('unexpected end of file')) {
+      throw new Error('Database is corrupt or too large. Try unzipping outside of VS Code and importing the unzipped folder instead.');
+    } else {
+      // delegate
+      throw e;
+    }
+  }
+}
+
+/**
+ * Fetches an archive database. The database might be on the internet
+ * or in the local filesystem.
+ *
+ * @param databaseUrl URL from which to grab the database
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ * @param progressCallback optional callback to send progress messages to
+ */
+async function databaseArchiveFetcher(
+  databaseUrl: string,
+  databasesManager: DatabaseManager,
+  storagePath: string,
+  progressCallback?: ProgressCallback
+): Promise<DatabaseItem> {
+  progressCallback?.({
+    message: 'Getting database',
+    step: 1,
+    maxStep: 4,
+  });
+  if (!storagePath) {
+    throw new Error('No storage path specified.');
+  }
+  await fs.ensureDir(storagePath);
+  const unzipPath = await getStorageFolder(storagePath, databaseUrl);
+
+  if (isFile(databaseUrl)) {
+    await readAndUnzip(databaseUrl, unzipPath);
+  } else {
+    await fetchAndUnzip(databaseUrl, unzipPath, progressCallback);
+  }
+
+  progressCallback?.({
+    message: 'Opening database',
+    step: 3,
+    maxStep: 4,
+  });
+
+  // find the path to the database. The actual database might be in a sub-folder
+  const dbPath = await findDirWithFile(
+    unzipPath,
+    '.dbinfo',
+    'codeql-database.yml'
+  );
+  if (dbPath) {
+    progressCallback?.({
+      message: 'Validating and fixing source location',
+      step: 4,
+      maxStep: 4,
+    });
+    await ensureZippedSourceLocation(dbPath);
+
+    const item = await databasesManager.openDatabase(Uri.file(dbPath));
+    databasesManager.setCurrentDatabaseItem(item);
+    return item;
+  } else {
+    throw new Error('Database not found in archive.');
+  }
+}
+
+async function getStorageFolder(storagePath: string, urlStr: string) {
+  // we need to generate a folder name for the unzipped archive,
+  // this needs to be human readable since we may use this name as the initial
+  // name for the database
+  const url = Uri.parse(urlStr);
+  // MacOS has a max filename length of 255
+  // and remove a few extra chars in case we need to add a counter at the end.
+  let lastName = path.basename(url.path).substring(0, 250);
+  if (lastName.endsWith('.zip')) {
+    lastName = lastName.substring(0, lastName.length - 4);
+  }
+
+  const realpath = await fs.realpath(storagePath);
+  let folderName = path.join(realpath, lastName);
+
+  // avoid overwriting existing folders
+  let counter = 0;
+  while (await fs.pathExists(folderName)) {
+    counter++;
+    folderName = path.join(realpath, `${lastName}-${counter}`);
+    if (counter > 100) {
+      throw new Error('Could not find a unique name for downloaded database.');
+    }
+  }
+  return folderName;
+}
+
+function validateHttpsUrl(databaseUrl: string) {
+  let uri;
+  try {
+    uri = Uri.parse(databaseUrl, true);
+  } catch (e) {
+    throw new Error(`Invalid url: ${databaseUrl}`);
+  }
+
+  if (uri.scheme !== 'https') {
+    throw new Error('Must use https for downloading a database.');
+  }
+}
+
+async function readAndUnzip(databaseUrl: string, unzipPath: string) {
+  const databaseFile = Uri.parse(databaseUrl).fsPath;
+  const directory = await unzipper.Open.file(databaseFile);
+  await directory.extract({ path: unzipPath });
+}
+
+async function fetchAndUnzip(
+  databaseUrl: string,
+  unzipPath: string,
+  progressCallback?: ProgressCallback
+) {
+  const response = await fetch(databaseUrl);
+
+  await checkForFailingResponse(response);
+
+  const unzipStream = unzipper.Extract({
+    path: unzipPath,
+  });
+
+  progressCallback?.({
+    maxStep: 3,
+    message: 'Unzipping database',
+    step: 2,
+  });
+  await new Promise((resolve, reject) => {
+    const handler = (err: Error) => {
+      if (err.message.startsWith('invalid signature')) {
+        reject(new Error('Not a valid archive.'));
+      } else {
+        reject(err);
+      }
+    };
+    response.body.on('error', handler);
+    unzipStream.on('error', handler);
+    unzipStream.on('close', resolve);
+    response.body.pipe(unzipStream);
+  });
+}
+
+async function checkForFailingResponse(response: Response): Promise<void | never> {
+  if (response.ok) {
+    return;
+  }
+
+  // An error downloading the database. Attempt to extract the resaon behind it.
+  const text = await response.text();
+  let msg: string;
+  try {
+    const obj = JSON.parse(text);
+    msg = obj.error || obj.message || obj.reason || JSON.stringify(obj, null, 2);
+  } catch (e) {
+    msg = text;
+  }
+  throw new Error(`Error downloading database.\n\nReason: ${msg}`);
+}
+
+function isFile(databaseUrl: string) {
+  return Uri.parse(databaseUrl).scheme === 'file';
+}
+
+/**
+ * Recursively looks for a file in a directory. If the file exists, then returns the directory containing the file.
+ *
+ * @param dir The directory to search
+ * @param toFind The file to recursively look for in this directory
+ *
+ * @returns the directory containing the file, or undefined if not found.
+ */
+// exported for testing
+export async function findDirWithFile(
+  dir: string,
+  ...toFind: string[]
+): Promise<string | undefined> {
+  if (!(await fs.stat(dir)).isDirectory()) {
+    return;
+  }
+  const files = await fs.readdir(dir);
+  if (toFind.some((file) => files.includes(file))) {
+    return dir;
+  }
+  for (const file of files) {
+    const newPath = path.join(dir, file);
+    const result = await findDirWithFile(newPath, ...toFind);
+    if (result) {
+      return result;
+    }
+  }
+  return;
+}
+
+/**
+ * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
+ * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
+ *
+ * After the {provider}/{org}/{name} path components, there may be the components
+ * related to sub pages.
+ *
+ * This function accepts any url that matches the patter above
+ *
+ * @param lgtmUrl The URL to the lgtm project
+ *
+ * @return true if this looks like an LGTM project url
+ */
+// exported for testing
+export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string {
+  if (!lgtmUrl) {
+    return false;
+  }
+
+  try {
+    const uri = Uri.parse(lgtmUrl, true);
+    if (uri.scheme !== 'https') {
+      return false;
+    }
+
+    if (uri.authority !== 'lgtm.com' && uri.authority !== 'www.lgtm.com') {
+      return false;
+    }
+
+    const paths = uri.path.split('/').filter((segment) => segment);
+    return paths.length >= 4 && paths[0] === 'projects';
+  } catch (e) {
+    return false;
+  }
+}
+
+// exported for testing
+export async function convertToDatabaseUrl(lgtmUrl: string) {
+  try {
+    const uri = Uri.parse(lgtmUrl, true);
+    const paths = ['api', 'v1.0'].concat(
+      uri.path.split('/').filter((segment) => segment)
+    ).slice(0, 6);
+    const projectUrl = `https://lgtm.com/${paths.join('/')}`;
+    const projectResponse = await fetch(projectUrl);
+    const projectJson = await projectResponse.json();
+
+    if (projectJson.code === 404) {
+      throw new Error();
+    }
+
+    const language = await promptForLanguage(projectJson);
+    if (!language) {
+      return;
+    }
+    return `https://lgtm.com/${[
+      'api',
+      'v1.0',
+      'snapshots',
+      projectJson.id,
+      language,
+    ].join('/')}`;
+  } catch (e) {
+    logger.log(`Error: ${e.message}`);
+    throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+  }
+}
+
+async function promptForLanguage(
+  projectJson: any
+): Promise<string | undefined> {
+  if (!projectJson?.languages?.length) {
+    return;
+  }
+  if (projectJson.languages.length === 1) {
+    return projectJson.languages[0].language;
+  }
+
+  return await window.showQuickPick(
+    projectJson.languages.map((lang: { language: string }) => lang.language), {
+    placeHolder: 'Select the database language to download:'
+  }
+  );
+}
+
+/**
+ * Databases created by the old odasa tool will not have a zipped
+ * source location. However, this extension works better if sources
+ * are zipped.
+ *
+ * This function ensures that the source location is zipped. If the
+ * `src` folder exists and the `src.zip` file does not, the `src`
+ * folder will be zipped and then deleted.
+ *
+ * @param databasePath The full path to the unzipped database
+ */
+async function ensureZippedSourceLocation(databasePath: string): Promise<void> {
+  const srcFolderPath = path.join(databasePath, 'src');
+  const srcZipPath = srcFolderPath + '.zip';
+
+  if ((await fs.pathExists(srcFolderPath)) && !(await fs.pathExists(srcZipPath))) {
+    await zip(srcFolderPath, srcZipPath);
+    await fs.remove(srcFolderPath);
+  }
+}

extensions/ql-vscode/src/databases-ui.ts

@@ -1,21 +1,50 @@
 import * as path from 'path';
-import { DisposableObject } from "semmle-vscode-utils";
-import { commands, Event, EventEmitter, ExtensionContext, ProviderResult, TreeDataProvider, TreeItem, Uri, window } from "vscode";
-import * as cli from './cli';
-import { DatabaseItem, DatabaseManager, getUpgradesDirectories } from "./databases";
-import { logger } from "./logging";
-import { clearCacheInDatabase, upgradeDatabase, UserCancellationException } from "./queries";
-import * as qsClient from './queryserver-client';
-import { getOnDiskWorkspaceFolders } from "./helpers";
+import { DisposableObject } from './vscode-utils/disposable-object';
+import {
+  Event,
+  EventEmitter,
+  ProviderResult,
+  TreeDataProvider,
+  TreeItem,
+  Uri,
+  window,
+  env
+} from 'vscode';
+import * as fs from 'fs-extra';
 
-type ThemableIconPath = { light: string, dark: string } | string;
+import * as cli from './cli';
+import {
+  DatabaseChangedEvent,
+  DatabaseItem,
+  DatabaseManager,
+  getUpgradesDirectories,
+} from './databases';
+import {
+  commandRunner,
+  commandRunnerWithProgress,
+  getOnDiskWorkspaceFolders,
+  ProgressCallback,
+  showAndLogErrorMessage
+} from './helpers';
+import { logger } from './logging';
+import { clearCacheInDatabase } from './run-queries';
+import * as qsClient from './queryserver-client';
+import { upgradeDatabase } from './upgrades';
+import {
+  importArchiveDatabase,
+  promptImportInternetDatabase,
+  promptImportLgtmDatabase,
+} from './databaseFetcher';
+import { CancellationToken } from 'vscode-jsonrpc';
+
+type ThemableIconPath = { light: string; dark: string } | string;
 
 /**
  * Path to icons to display next to currently selected database.
  */
 const SELECTED_DATABASE_ICON: ThemableIconPath = {
-  light: 'media/check-light-mode.svg',
-  dark: 'media/check-dark-mode.svg',
+  light: 'media/light/check.svg',
+  dark: 'media/dark/check.svg',
 };
 
 /**
@@ -23,14 +52,23 @@ const SELECTED_DATABASE_ICON: ThemableIconPath = {
  */
 const INVALID_DATABASE_ICON: ThemableIconPath = 'media/red-x.svg';
 
-function joinThemableIconPath(base: string, iconPath: ThemableIconPath): ThemableIconPath {
+function joinThemableIconPath(
+  base: string,
+  iconPath: ThemableIconPath
+): ThemableIconPath {
   if (typeof iconPath == 'object')
     return {
       light: path.join(base, iconPath.light),
-      dark: path.join(base, iconPath.dark)
+      dark: path.join(base, iconPath.dark),
     };
-  else
-    return path.join(base, iconPath);
+  else return path.join(base, iconPath);
+}
+
+enum SortOrder {
+  NameAsc = 'NameAsc',
+  NameDesc = 'NameDesc',
+  DateAddedAsc = 'DateAddedAsc',
+  DateAddedDesc = 'DateAddedDesc',
 }
 
 /**
@@ -38,44 +76,68 @@ function joinThemableIconPath(base: string, iconPath: ThemableIconPath): Themabl
  */
 class DatabaseTreeDataProvider extends DisposableObject
   implements TreeDataProvider<DatabaseItem> {
+  private _sortOrder = SortOrder.NameAsc;
 
   private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
   private currentDatabaseItem: DatabaseItem | undefined;
 
-  constructor(private ctx: ExtensionContext, private databaseManager: DatabaseManager) {
+  constructor(
+    private databaseManager: DatabaseManager,
+    private readonly extensionPath: string
+  ) {
     super();
 
     this.currentDatabaseItem = databaseManager.currentDatabaseItem;
 
-    this.push(this.databaseManager.onDidChangeDatabaseItem(this.handleDidChangeDatabaseItem));
-    this.push(this.databaseManager.onDidChangeCurrentDatabaseItem(
-      this.handleDidChangeCurrentDatabaseItem));
+    this.push(
+      this.databaseManager.onDidChangeDatabaseItem(
+        this.handleDidChangeDatabaseItem
+      )
+    );
+    this.push(
+      this.databaseManager.onDidChangeCurrentDatabaseItem(
+        this.handleDidChangeCurrentDatabaseItem
+      )
+    );
   }
 
   public get onDidChangeTreeData(): Event<DatabaseItem | undefined> {
     return this._onDidChangeTreeData.event;
   }
 
-  private handleDidChangeDatabaseItem = (databaseItem: DatabaseItem | undefined): void => {
-    this._onDidChangeTreeData.fire(databaseItem);
-  }
+  private handleDidChangeDatabaseItem = (event: DatabaseChangedEvent): void => {
+    // Note that events from the databse manager are instances of DatabaseChangedEvent
+    // and events fired by the UI are instances of DatabaseItem
 
-  private handleDidChangeCurrentDatabaseItem = (databaseItem: DatabaseItem | undefined): void => {
+    // When event.item is undefined, then the entire tree is refreshed.
+    // When event.item is a db item, then only that item is refreshed.
+    this._onDidChangeTreeData.fire(event.item);
+  };
+
+  private handleDidChangeCurrentDatabaseItem = (
+    event: DatabaseChangedEvent
+  ): void => {
     if (this.currentDatabaseItem) {
       this._onDidChangeTreeData.fire(this.currentDatabaseItem);
     }
-    this.currentDatabaseItem = databaseItem;
+    this.currentDatabaseItem = event.item;
     if (this.currentDatabaseItem) {
       this._onDidChangeTreeData.fire(this.currentDatabaseItem);
     }
-  }
+  };
 
   public getTreeItem(element: DatabaseItem): TreeItem {
     const item = new TreeItem(element.name);
     if (element === this.currentDatabaseItem) {
-      item.iconPath = joinThemableIconPath(this.ctx.extensionPath, SELECTED_DATABASE_ICON);
+      item.iconPath = joinThemableIconPath(
+        this.extensionPath,
+        SELECTED_DATABASE_ICON
+      );
     } else if (element.error !== undefined) {
-      item.iconPath = joinThemableIconPath(this.ctx.extensionPath, INVALID_DATABASE_ICON);
+      item.iconPath = joinThemableIconPath(
+        this.extensionPath,
+        INVALID_DATABASE_ICON
+      );
     }
     item.tooltip = element.databaseUri.fsPath;
     return item;
@@ -83,9 +145,19 @@ class DatabaseTreeDataProvider extends DisposableObject
 
   public getChildren(element?: DatabaseItem): ProviderResult<DatabaseItem[]> {
     if (element === undefined) {
-      return this.databaseManager.databaseItems.slice(0);
-    }
-    else {
+      return this.databaseManager.databaseItems.slice(0).sort((db1, db2) => {
+        switch (this.sortOrder) {
+          case SortOrder.NameAsc:
+            return db1.name.localeCompare(db2.name, env.language);
+          case SortOrder.NameDesc:
+            return db2.name.localeCompare(db1.name, env.language);
+          case SortOrder.DateAddedAsc:
+            return (db1.dateAdded || 0) - (db2.dateAdded || 0);
+          case SortOrder.DateAddedDesc:
+            return (db2.dateAdded || 0) - (db1.dateAdded || 0);
+        }
+      });
+    } else {
       return [];
     }
   }
@@ -97,14 +169,22 @@ class DatabaseTreeDataProvider extends DisposableObject
   public getCurrent(): DatabaseItem | undefined {
     return this.currentDatabaseItem;
   }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire();
+  }
 }
 
 /** Gets the first element in the given list, if any, or undefined if the list is empty or undefined. */
 function getFirst(list: Uri[] | undefined): Uri | undefined {
   if (list === undefined || list.length === 0) {
     return undefined;
-  }
-  else {
+  } else {
     return list[0];
   }
 }
@@ -117,62 +197,271 @@ function getFirst(list: Uri[] | undefined): Uri | undefined {
  * XXX: no validation is done other than checking the directory name
  * to make sure it really is a database directory.
  */
-async function chooseDatabaseDir(): Promise<Uri | undefined> {
+async function chooseDatabaseDir(byFolder: boolean): Promise<Uri | undefined> {
   const chosen = await window.showOpenDialog({
-    openLabel: 'Choose Database',
-    canSelectFiles: true,
-    canSelectFolders: true,
-    canSelectMany: false
+    openLabel: byFolder ? 'Choose Database folder' : 'Choose Database archive',
+    canSelectFiles: !byFolder,
+    canSelectFolders: byFolder,
+    canSelectMany: false,
+    filters: byFolder ? {} : { Archives: ['zip'] },
   });
   return getFirst(chosen);
 }
 
 export class DatabaseUI extends DisposableObject {
-  public constructor(ctx: ExtensionContext, private cliserver: cli.CodeQLCliServer, private databaseManager: DatabaseManager,
-    private readonly queryServer: qsClient.QueryServerClient | undefined) {
+  private treeDataProvider: DatabaseTreeDataProvider;
 
+  public constructor(
+    private cliserver: cli.CodeQLCliServer,
+    private databaseManager: DatabaseManager,
+    private readonly queryServer: qsClient.QueryServerClient | undefined,
+    private readonly storagePath: string,
+    readonly extensionPath: string
+  ) {
     super();
 
-    const treeDataProvider = this.push(new DatabaseTreeDataProvider(ctx, databaseManager));
-    this.push(window.createTreeView('codeQLDatabases', { treeDataProvider }));
+    this.treeDataProvider = this.push(
+      new DatabaseTreeDataProvider(databaseManager, extensionPath)
+    );
+    this.push(
+      window.createTreeView('codeQLDatabases', {
+        treeDataProvider: this.treeDataProvider,
+        canSelectMany: true,
+      })
+    );
 
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabase', this.handleChooseDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.setCurrentDatabase', this.handleSetCurrentDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.upgradeCurrentDatabase', this.handleUpgradeCurrentDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.clearCache', this.handleClearCache));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.setCurrentDatabase', this.handleMakeCurrentDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.removeDatabase', this.handleRemoveDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.upgradeDatabase', this.handleUpgradeDatabase));
+    logger.log('Registering database panel commands.');
+    this.push(
+      commandRunnerWithProgress(
+        'codeQL.setCurrentDatabase',
+        this.handleSetCurrentDatabase,
+        {
+          title: 'Importing database from archive',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQL.upgradeCurrentDatabase',
+        this.handleUpgradeCurrentDatabase,
+        {
+          title: 'Upgrading current database',
+          cancellable: true,
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQL.clearCache',
+        this.handleClearCache,
+        {
+          title: 'Clearing Cache',
+        })
+    );
+
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseFolder',
+        this.handleChooseDatabaseFolder,
+        {
+          title: 'Adding database from folder',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseArchive',
+        this.handleChooseDatabaseArchive,
+        {
+          title: 'Adding database from archive',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseInternet',
+        this.handleChooseDatabaseInternet,
+        {
+          title: 'Adding database from URL',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseLgtm',
+        this.handleChooseDatabaseLgtm,
+        {
+          title: 'Adding database from LGTM',
+        })
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.setCurrentDatabase',
+        this.handleMakeCurrentDatabase
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.sortByName',
+        this.handleSortByName
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.sortByDateAdded',
+        this.handleSortByDateAdded
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.removeDatabase',
+        this.handleRemoveDatabase
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.upgradeDatabase',
+        this.handleUpgradeDatabase,
+        {
+          title: 'Upgrading database',
+          cancellable: true,
+        }
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.renameDatabase',
+        this.handleRenameDatabase
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.openDatabaseFolder',
+        this.handleOpenFolder
+      )
+    );
   }
 
-  private handleMakeCurrentDatabase = async (databaseItem: DatabaseItem): Promise<void> => {
+  private handleMakeCurrentDatabase = async (
+    databaseItem: DatabaseItem
+  ): Promise<void> => {
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
+  };
+
+  handleChooseDatabaseFolder = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
+    try {
+      return await this.chooseAndSetDatabase(true, progress, token);
+    } catch (e) {
+      showAndLogErrorMessage(e.message);
+      return undefined;
+    }
+  };
+
+  handleChooseDatabaseArchive = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
+    try {
+      return await this.chooseAndSetDatabase(false, progress, token);
+    } catch (e) {
+      showAndLogErrorMessage(e.message);
+      return undefined;
+    }
+  };
+
+  handleChooseDatabaseInternet = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<
+    DatabaseItem | undefined
+  > => {
+    return await promptImportInternetDatabase(
+      this.databaseManager,
+      this.storagePath,
+      progress,
+      token
+    );
+  };
+
+  handleChooseDatabaseLgtm = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
+    return await promptImportLgtmDatabase(
+      this.databaseManager,
+      this.storagePath,
+      progress,
+      token
+    );
+  };
+
+  async tryUpgradeCurrentDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ) {
+    await this.handleUpgradeCurrentDatabase(progress, token);
   }
 
-  private handleChooseDatabase = async (): Promise<DatabaseItem | undefined> => {
-    return await this.chooseAndSetDatabase();
-  }
+  private handleSortByName = async () => {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  };
 
-  private handleUpgradeCurrentDatabase = async (): Promise<void> => {
-    await this.handleUpgradeDatabase(this.databaseManager.currentDatabaseItem);
-  }
+  private handleSortByDateAdded = async () => {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAddedAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateAddedDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAddedAsc;
+    }
+  };
 
-  private handleUpgradeDatabase = async (databaseItem: DatabaseItem | undefined): Promise<void> => {
+  private handleUpgradeCurrentDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> => {
+    await this.handleUpgradeDatabase(
+      progress, token,
+      this.databaseManager.currentDatabaseItem,
+      []
+    );
+  };
+
+  private handleUpgradeDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+    databaseItem: DatabaseItem | undefined,
+    multiSelect: DatabaseItem[] | undefined,
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      await Promise.all(
+        multiSelect.map((dbItem) => this.handleUpgradeDatabase(progress, token, dbItem, []))
+      );
+    }
     if (this.queryServer === undefined) {
-      logger.log('Received request to upgrade database, but there is no running query server.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but there is no running query server.'
+      );
     }
     if (databaseItem === undefined) {
-      logger.log('Received request to upgrade database, but no database was provided.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but no database was provided.'
+      );
     }
     if (databaseItem.contents === undefined) {
-      logger.log('Received request to upgrade database, but database contents could not be found.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but database contents could not be found.'
+      );
     }
     if (databaseItem.contents.dbSchemeUri === undefined) {
-      logger.log('Received request to upgrade database, but database has no schema.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but database has no schema.'
+      );
     }
 
     // Search for upgrade scripts in any workspace folders available
@@ -180,59 +469,132 @@ export class DatabaseUI extends DisposableObject {
 
     const upgradeInfo = await this.cliserver.resolveUpgrades(
       databaseItem.contents.dbSchemeUri.fsPath,
-      searchPath,
+      searchPath
     );
 
     const { scripts, finalDbscheme } = upgradeInfo;
 
     if (finalDbscheme === undefined) {
-      logger.log('Could not determine target dbscheme to upgrade to.');
-      return;
+      throw new Error('Could not determine target dbscheme to upgrade to.');
     }
     const targetDbSchemeUri = Uri.file(finalDbscheme);
 
+    await upgradeDatabase(
+      this.queryServer,
+      databaseItem,
+      targetDbSchemeUri,
+      getUpgradesDirectories(scripts),
+      progress,
+      token
+    );
+  };
+
+  private handleClearCache = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> => {
+    if (
+      this.queryServer !== undefined &&
+      this.databaseManager.currentDatabaseItem !== undefined
+    ) {
+      await clearCacheInDatabase(
+        this.queryServer,
+        this.databaseManager.currentDatabaseItem,
+        progress,
+        token
+      );
+    }
+  };
+
+  private handleSetCurrentDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+    uri: Uri,
+  ): Promise<void> => {
     try {
-      await upgradeDatabase(this.queryServer, databaseItem, targetDbSchemeUri, getUpgradesDirectories(scripts));
-    }
-    catch (e) {
-      if (e instanceof UserCancellationException) {
-        logger.log(e.message);
+      // Assume user has selected an archive if the file has a .zip extension
+      if (uri.path.endsWith('.zip')) {
+        await importArchiveDatabase(
+          uri.toString(true),
+          this.databaseManager,
+          this.storagePath,
+          progress,
+          token
+        );
+      } else {
+        await this.setCurrentDatabase(uri);
       }
-      else
-        throw e;
+    } catch (e) {
+      // rethrow and let this be handled by default error handling.
+      throw new Error(
+        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${
+        e.message
+        }`
+      );
     }
-  }
+  };
 
-  private handleClearCache = async (): Promise<void> => {
-    if ((this.queryServer !== undefined) &&
-      (this.databaseManager.currentDatabaseItem !== undefined)) {
-
-      await clearCacheInDatabase(this.queryServer, this.databaseManager.currentDatabaseItem);
+  private handleRemoveDatabase = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      multiSelect.forEach((dbItem) =>
+        this.databaseManager.removeDatabaseItem(dbItem)
+      );
+    } else {
+      this.databaseManager.removeDatabaseItem(databaseItem);
     }
-  }
+  };
 
-  private handleSetCurrentDatabase = async (uri: Uri): Promise<DatabaseItem | undefined> => {
-    return await this.setCurrentDatabase(uri);
-  }
+  private handleRenameDatabase = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    this.assertSingleDatabase(multiSelect);
 
-  private handleRemoveDatabase = (databaseItem: DatabaseItem): void => {
-    this.databaseManager.removeDatabaseItem(databaseItem);
-  }
+    const newName = await window.showInputBox({
+      prompt: 'Choose new database name',
+      value: databaseItem.name,
+    });
+
+    if (newName) {
+      this.databaseManager.renameDatabaseItem(databaseItem, newName);
+    }
+  };
+
+  private handleOpenFolder = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      await Promise.all(
+        multiSelect.map((dbItem) => env.openExternal(dbItem.databaseUri))
+      );
+    } else {
+      await env.openExternal(databaseItem.databaseUri);
+    }
+  };
 
   /**
    * Return the current database directory. If we don't already have a
    * current database, ask the user for one, and return that, or
    * undefined if they cancel.
    */
-  public async getDatabaseItem(): Promise<DatabaseItem | undefined> {
+  public async getDatabaseItem(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> {
     if (this.databaseManager.currentDatabaseItem === undefined) {
-      await this.chooseAndSetDatabase();
+      await this.chooseAndSetDatabase(false, progress, token);
     }
 
     return this.databaseManager.currentDatabaseItem;
   }
 
-  private async setCurrentDatabase(uri: Uri): Promise<DatabaseItem | undefined> {
+  private async setCurrentDatabase(
+    uri: Uri
+  ): Promise<DatabaseItem | undefined> {
     let databaseItem = this.databaseManager.findDatabaseItem(uri);
     if (databaseItem === undefined) {
       databaseItem = await this.databaseManager.openDatabase(uri);
@@ -246,13 +608,69 @@ export class DatabaseUI extends DisposableObject {
    * Ask the user for a database directory. Returns the chosen database, or `undefined` if the
    * operation was canceled.
    */
-  private async chooseAndSetDatabase(): Promise<DatabaseItem | undefined> {
-    const uri = await chooseDatabaseDir();
-    if (uri !== undefined) {
-      return await this.setCurrentDatabase(uri);
-    }
-    else {
+  private async chooseAndSetDatabase(
+    byFolder: boolean,
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<DatabaseItem | undefined> {
+    const uri = await chooseDatabaseDir(byFolder);
+
+    if (!uri) {
       return undefined;
     }
+
+    if (byFolder) {
+      const fixedUri = await this.fixDbUri(uri);
+      // we are selecting a database folder
+      return await this.setCurrentDatabase(fixedUri);
+    } else {
+      // we are selecting a database archive. Must unzip into a workspace-controlled area
+      // before importing.
+      return await importArchiveDatabase(
+        uri.toString(true),
+        this.databaseManager,
+        this.storagePath,
+        progress,
+        token
+      );
+    }
+  }
+
+  /**
+   * Perform some heuristics to ensure a proper database location is chosen.
+   *
+   * 1. If the selected URI to add is a file, choose the containing directory
+   * 2. If the selected URI is a directory matching db-*, choose the containing directory
+   * 3. choose the current directory
+   *
+   * @param uri a URI that is a datbase folder or inside it
+   *
+   * @return the actual database folder found by using the heuristics above.
+   */
+  private async fixDbUri(uri: Uri): Promise<Uri> {
+    let dbPath = uri.fsPath;
+    if ((await fs.stat(dbPath)).isFile()) {
+      dbPath = path.dirname(dbPath);
+    }
+
+    if (isLikelyDbFolder(dbPath)) {
+      dbPath = path.dirname(dbPath);
+    }
+    return Uri.file(dbPath);
+  }
+
+  private assertSingleDatabase(
+    multiSelect: DatabaseItem[] = [],
+    message = 'Please select a single database.'
+  ) {
+    if (multiSelect.length > 1) {
+      throw new Error(message);
+    }
   }
 }
+
+// TODO: Get the list of supported languages from a list that will be auto-updated.
+const dbRegeEx = /^db-(javascript|go|cpp|java|python|csharp)$/;
+function isLikelyDbFolder(dbPath: string) {
+  return path.basename(dbPath).match(dbRegeEx);
+}

extensions/ql-vscode/src/databases.ts

@@ -5,8 +5,8 @@ import * as vscode from 'vscode';
 import * as cli from './cli';
 import { ExtensionContext } from 'vscode';
 import { showAndLogErrorMessage, showAndLogWarningMessage, showAndLogInformationMessage } from './helpers';
-import { zipArchiveScheme, encodeSourceArchiveUri, decodeSourceArchiveUri } from './archive-filesystem-provider';
-import { DisposableObject } from 'semmle-vscode-utils';
+import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
+import { DisposableObject } from './vscode-utils/disposable-object';
 import { QueryServerConfig } from './config';
 import { Logger, logger } from './logging';
 
@@ -24,21 +24,23 @@ import { Logger, logger } from './logging';
  * The name of the key in the workspaceState dictionary in which we
  * persist the current database across sessions.
  */
-const CURRENT_DB: string = 'currentDatabase';
+const CURRENT_DB = 'currentDatabase';
 
 /**
  * The name of the key in the workspaceState dictionary in which we
  * persist the list of databases across sessions.
  */
-const DB_LIST: string = 'databaseList';
+const DB_LIST = 'databaseList';
 
 export interface DatabaseOptions {
   displayName?: string;
   ignoreSourceArchive?: boolean;
+  dateAdded?: number | undefined;
 }
 
 interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
+  dateAdded: number | undefined;
 }
 
 interface PersistedDatabaseItem {
@@ -107,10 +109,11 @@ async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
   return vscode.Uri.file(dbAbsolutePath);
 }
 
-async function findSourceArchive(databasePath: string, silent: boolean = false):
-  Promise<vscode.Uri | undefined> {
+async function findSourceArchive(
+  databasePath: string, silent = false
+): Promise<vscode.Uri | undefined> {
 
-  const relativePaths = ['src', 'output/src_archive']
+  const relativePaths = ['src', 'output/src_archive'];
 
   for (const relativePath of relativePaths) {
     const basePath = path.join(databasePath, relativePath);
@@ -118,18 +121,21 @@ async function findSourceArchive(databasePath: string, silent: boolean = false):
 
     if (await fs.pathExists(basePath)) {
       return vscode.Uri.file(basePath);
-    }
-    else if (await fs.pathExists(zipPath)) {
-      return vscode.Uri.file(zipPath).with({ scheme: zipArchiveScheme });
+    } else if (await fs.pathExists(zipPath)) {
+      return encodeArchiveBasePath(zipPath);
     }
   }
-  if (!silent)
-    showAndLogInformationMessage(`Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`);
+  if (!silent) {
+    showAndLogInformationMessage(
+      `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`
+    );
+  }
   return undefined;
 }
 
-async function resolveDatabase(databasePath: string):
-  Promise<DatabaseContents | undefined> {
+async function resolveDatabase(
+  databasePath: string
+): Promise<DatabaseContents> {
 
   const name = path.basename(databasePath);
 
@@ -151,20 +157,6 @@ async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
   return await glob('*.dbscheme', { cwd: dbDirectory });
 }
 
-async function resolveRawDataset(datasetPath: string): Promise<DatabaseContents | undefined> {
-  if ((await getDbSchemeFiles(datasetPath)).length > 0) {
-    return {
-      kind: DatabaseKind.RawDataset,
-      name: path.basename(datasetPath),
-      datasetUri: vscode.Uri.file(datasetPath),
-      sourceArchiveUri: undefined
-    };
-  }
-  else {
-    return undefined;
-  }
-}
-
 async function resolveDatabaseContents(uri: vscode.Uri): Promise<DatabaseContents> {
   if (uri.scheme !== 'file') {
     throw new Error(`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`);
@@ -174,7 +166,7 @@ async function resolveDatabaseContents(uri: vscode.Uri): Promise<DatabaseContent
     throw new InvalidDatabaseError(`Database '${databasePath}' does not exist.`);
   }
 
-  const contents = await resolveDatabase(databasePath) || await resolveRawDataset(databasePath);
+  const contents = await resolveDatabase(databasePath);
 
   if (contents === undefined) {
     throw new InvalidDatabaseError(`'${databasePath}' is not a valid database.`);
@@ -200,7 +192,7 @@ export interface DatabaseItem {
   /** The URI of the database */
   readonly databaseUri: vscode.Uri;
   /** The name of the database to be displayed in the UI */
-  readonly name: string;
+  name: string;
   /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
   readonly sourceArchive: vscode.Uri | undefined;
   /**
@@ -208,6 +200,12 @@ export interface DatabaseItem {
    * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`.
    */
   readonly contents: DatabaseContents | undefined;
+
+  /**
+   * The date this database was added as a unix timestamp. Or undefined if we don't know.
+   */
+  readonly dateAdded: number | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
   /**
@@ -236,6 +234,11 @@ export interface DatabaseItem {
    */
   getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
 
+  /**
+   * Returns dataset folder of exported database.
+   */
+  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive,
    * as displayed in the filesystem explorer.
@@ -248,16 +251,37 @@ export interface DatabaseItem {
   belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
 }
 
-class DatabaseItemImpl implements DatabaseItem {
+export enum DatabaseEventKind {
+  Add = 'Add',
+  Remove = 'Remove',
+
+  // Fired when databases are refreshed from persisted state
+  Refresh = 'Refresh',
+
+  // Fired when the current database changes
+  Change = 'Change',
+
+  Rename = 'Rename'
+}
+
+export interface DatabaseChangedEvent {
+  kind: DatabaseEventKind;
+  item: DatabaseItem | undefined;
+}
+
+// Exported for testing
+export class DatabaseItemImpl implements DatabaseItem {
   private _error: Error | undefined = undefined;
   private _contents: DatabaseContents | undefined;
   /** A cache of database info */
   private _dbinfo: cli.DbInfo | undefined;
 
-  public constructor(public readonly databaseUri: vscode.Uri,
-    contents: DatabaseContents | undefined, private options: FullDatabaseOptions,
-    private readonly onChanged: (item: DatabaseItemImpl) => void) {
-
+  public constructor(
+    public readonly databaseUri: vscode.Uri,
+    contents: DatabaseContents | undefined,
+    private options: FullDatabaseOptions,
+    private readonly onChanged: (event: DatabaseChangedEvent) => void
+  ) {
     this._contents = contents;
   }
 
@@ -273,11 +297,14 @@ class DatabaseItemImpl implements DatabaseItem {
     }
   }
 
+  public set name(newName: string) {
+    this.options.displayName = newName;
+  }
+
   public get sourceArchive(): vscode.Uri | undefined {
     if (this.options.ignoreSourceArchive || (this._contents === undefined)) {
       return undefined;
-    }
-    else {
+    } else {
       return this._contents.sourceArchiveUri;
     }
   }
@@ -286,6 +313,10 @@ class DatabaseItemImpl implements DatabaseItem {
     return this._contents;
   }
 
+  public get dateAdded(): number | undefined {
+    return this.options.dateAdded;
+  }
+
   public get error(): Error | undefined {
     return this._error;
   }
@@ -303,46 +334,52 @@ class DatabaseItemImpl implements DatabaseItem {
       }
     }
     finally {
-      this.onChanged(this);
+      this.onChanged({
+        kind: DatabaseEventKind.Refresh,
+        item: this
+      });
     }
   }
 
-  public resolveSourceFile(file: string | undefined): vscode.Uri {
+  public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
-    if (sourceArchive === undefined) {
-      if (file !== undefined) {
-        // Treat it as an absolute path.
-        return vscode.Uri.file(file);
-      }
-      else {
+    const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
+    if (uri && uri.scheme !== 'file') {
+      throw new Error(`Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`);
+    }
+    if (!sourceArchive) {
+      if (uri) {
+        return uri;
+      } else {
         return this.databaseUri;
       }
     }
-    else {
-      if (file !== undefined) {
-        const absoluteFilePath = file.replace(':', '_');
-        // Strip any leading slashes from the file path, and replace `:` with `_`.
-        const relativeFilePath = absoluteFilePath.replace(/^\/*/, '').replace(':', '_');
-        if (sourceArchive.scheme == zipArchiveScheme) {
-          return encodeSourceArchiveUri({
-            pathWithinSourceArchive: absoluteFilePath,
-            sourceArchiveZipPath: sourceArchive.fsPath,
-          });
-        }
-        else {
-          let newPath = sourceArchive.path;
-          if (!newPath.endsWith('/')) {
-            // Ensure a trailing slash.
-            newPath += '/';
-          }
-          newPath += relativeFilePath;
 
-          return sourceArchive.with({ path: newPath });
+    if (uri) {
+      const relativeFilePath = decodeURI(uri.path).replace(':', '_').replace(/^\/*/, '');
+      if (sourceArchive.scheme === zipArchiveScheme) {
+        const zipRef = decodeSourceArchiveUri(sourceArchive);
+        const pathWithinSourceArchive = zipRef.pathWithinSourceArchive === '/'
+          ? relativeFilePath
+          : zipRef.pathWithinSourceArchive + '/' + relativeFilePath;
+        return encodeSourceArchiveUri({
+          pathWithinSourceArchive,
+          sourceArchiveZipPath: zipRef.sourceArchiveZipPath,
+        });
+
+      } else {
+        let newPath = sourceArchive.path;
+        if (!newPath.endsWith('/')) {
+          // Ensure a trailing slash.
+          newPath += '/';
         }
+        newPath += relativeFilePath;
+
+        return sourceArchive.with({ path: newPath });
       }
-      else {
-        return sourceArchive;
-      }
+
+    } else {
+      return sourceArchive;
     }
   }
 
@@ -385,6 +422,14 @@ class DatabaseItemImpl implements DatabaseItem {
     return dbInfo.sourceLocationPrefix;
   }
 
+  /**
+   * Returns path to dataset folder of database.
+   */
+  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
+    const dbInfo = await this.getDbInfo(server);
+    return dbInfo.datasetFolder;
+  }
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive.
    */
@@ -392,10 +437,7 @@ class DatabaseItemImpl implements DatabaseItem {
     const sourceArchive = this.sourceArchive;
     if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith('.zip'))
       return undefined;
-    return encodeSourceArchiveUri({
-      pathWithinSourceArchive: '/',
-      sourceArchiveZipPath: sourceArchive.fsPath,
-    });
+    return encodeArchiveBasePath(sourceArchive.fsPath);
   }
 
   /**
@@ -414,46 +456,48 @@ class DatabaseItemImpl implements DatabaseItem {
  * `event` fires. If waiting for the event takes too long (by default
  * >1000ms) log a warning, and resolve to undefined.
  */
-function eventFired<T>(event: vscode.Event<T>, timeoutMs: number = 1000): Promise<T | undefined> {
+function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | undefined> {
   return new Promise((res, _rej) => {
-    let timeout: NodeJS.Timeout | undefined;
-    let disposable: vscode.Disposable | undefined;
-    function dispose() {
-      if (timeout !== undefined) clearTimeout(timeout);
-      if (disposable !== undefined) disposable.dispose();
-    }
-    disposable = event(e => {
-      res(e); dispose();
-    });
-    timeout = setTimeout(() => {
+    const timeout = setTimeout(() => {
       logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
-      res(undefined); dispose();
+      res(undefined);
+      dispose();
     }, timeoutMs);
+    const disposable = event(e => {
+      res(e);
+      dispose();
+    });
+    function dispose() {
+      clearTimeout(timeout);
+      disposable.dispose();
+    }
   });
 }
 
 export class DatabaseManager extends DisposableObject {
-  private readonly _onDidChangeDatabaseItem =
-    this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  private readonly _onDidChangeDatabaseItem = this.push(new vscode.EventEmitter<DatabaseChangedEvent>());
+
   readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
 
-  private readonly _onDidChangeCurrentDatabaseItem =
-    this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  private readonly _onDidChangeCurrentDatabaseItem = this.push(new vscode.EventEmitter<DatabaseChangedEvent>());
   readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event;
 
   private readonly _databaseItems: DatabaseItemImpl[] = [];
   private _currentDatabaseItem: DatabaseItem | undefined = undefined;
 
-  constructor(private ctx: ExtensionContext,
+  constructor(
+    private ctx: ExtensionContext,
     public config: QueryServerConfig,
-    public logger: Logger) {
+    public logger: Logger
+  ) {
     super();
 
     this.loadPersistedState();  // Let this run async.
   }
 
-  public async openDatabase(uri: vscode.Uri, options?: DatabaseOptions):
-    Promise<DatabaseItem> {
+  public async openDatabase(
+    uri: vscode.Uri, options?: DatabaseOptions
+  ): Promise<DatabaseItem> {
 
     const contents = await resolveDatabaseContents(uri);
     const realOptions = options || {};
@@ -462,10 +506,11 @@ export class DatabaseManager extends DisposableObject {
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ?
         realOptions.ignoreSourceArchive : isQLTestDatabase,
-      displayName: realOptions.displayName
+      displayName: realOptions.displayName,
+      dateAdded: realOptions.dateAdded || Date.now()
     };
-    const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (item) => {
-      this._onDidChangeDatabaseItem.fire(item);
+    const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => {
+      this._onDidChangeDatabaseItem.fire(event);
     });
     await this.addDatabaseItem(databaseItem);
     await this.addDatabaseSourceArchiveFolder(databaseItem);
@@ -512,11 +557,13 @@ export class DatabaseManager extends DisposableObject {
     }
   }
 
-  private async createDatabaseItemFromPersistedState(state: PersistedDatabaseItem):
-    Promise<DatabaseItem> {
+  private async createDatabaseItemFromPersistedState(
+    state: PersistedDatabaseItem
+  ): Promise<DatabaseItem> {
 
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
+    let dateAdded = undefined;
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -524,14 +571,18 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.ignoreSourceArchive === 'boolean') {
         ignoreSourceArchive = state.options.ignoreSourceArchive;
       }
+      if (typeof state.options.dateAdded === 'number') {
+        dateAdded = state.options.dateAdded;
+      }
     }
     const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: ignoreSourceArchive,
-      displayName: displayName
+      ignoreSourceArchive,
+      displayName,
+      dateAdded
     };
-    const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri), undefined, fullOptions,
-      (item) => {
-        this._onDidChangeDatabaseItem.fire(item)
+    const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri, true), undefined, fullOptions,
+      (event) => {
+        this._onDidChangeDatabaseItem.fire(event);
       });
     await this.addDatabaseItem(item);
 
@@ -558,7 +609,7 @@ export class DatabaseManager extends DisposableObject {
       }
     } catch (e) {
       // database list had an unexpected type - nothing to be done?
-      showAndLogErrorMessage('Database list loading failed: ${}', e.message);
+      showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
     }
   }
 
@@ -571,7 +622,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   public async setCurrentDatabaseItem(item: DatabaseItem | undefined,
-    skipRefresh: boolean = false): Promise<void> {
+    skipRefresh = false): Promise<void> {
 
     if (!skipRefresh && (item !== undefined)) {
       await item.refresh();  // Will throw on invalid database.
@@ -579,7 +630,10 @@ export class DatabaseManager extends DisposableObject {
     if (this._currentDatabaseItem !== item) {
       this._currentDatabaseItem = item;
       this.updatePersistedCurrentDatabaseItem();
-      this._onDidChangeCurrentDatabaseItem.fire(item);
+      this._onDidChangeCurrentDatabaseItem.fire({
+        item,
+        kind: DatabaseEventKind.Change
+      });
     }
   }
 
@@ -597,10 +651,28 @@ export class DatabaseManager extends DisposableObject {
     return this._databaseItems.find(item => item.databaseUri.toString(true) === uriString);
   }
 
+  public findDatabaseItemBySourceArchive(uri: vscode.Uri): DatabaseItem | undefined {
+    const uriString = uri.toString(true);
+    return this._databaseItems.find(item => item.sourceArchive && item.sourceArchive.toString(true) === uriString);
+  }
+
   private async addDatabaseItem(item: DatabaseItemImpl) {
     this._databaseItems.push(item);
     this.updatePersistedDatabaseList();
-    this._onDidChangeDatabaseItem.fire(undefined);
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Add
+    });
+  }
+
+  public async renameDatabaseItem(item: DatabaseItem, newName: string) {
+    item.name = newName;
+    this.updatePersistedDatabaseList();
+    this._onDidChangeDatabaseItem.fire({
+      item,
+      kind: DatabaseEventKind.Rename
+    });
   }
 
   public removeDatabaseItem(item: DatabaseItem) {
@@ -619,7 +691,19 @@ export class DatabaseManager extends DisposableObject {
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
-    this._onDidChangeDatabaseItem.fire(undefined);
+    // Delete folder from file system only if it is controlled by the extension
+    if (this.isExtensionControlledLocation(item.databaseUri)) {
+      logger.log('Deleting database from filesystem.');
+      fs.remove(item.databaseUri.path).then(
+        () => logger.log(`Deleted '${item.databaseUri.path}'`),
+        e => logger.log(`Failed to delete '${item.databaseUri.path}'. Reason: ${e.message}`));
+    }
+
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Remove
+    });
   }
 
   private updatePersistedCurrentDatabaseItem(): void {
@@ -630,6 +714,11 @@ export class DatabaseManager extends DisposableObject {
   private updatePersistedDatabaseList(): void {
     this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
   }
+
+  private isExtensionControlledLocation(uri: vscode.Uri) {
+    const storagePath = this.ctx.storagePath || this.ctx.globalStoragePath;
+    return uri.path.startsWith(storagePath);
+  }
 }
 
 /**

extensions/ql-vscode/src/discovery.ts

@@ -0,0 +1,93 @@
+import { DisposableObject } from './vscode-utils/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
+
+/**
+ * Base class for "discovery" operations, which scan the file system to find specific kinds of
+ * files. This class automatically prevents more than one discovery operation from running at the
+ * same time.
+ */
+export abstract class Discovery<T> extends DisposableObject {
+  private retry = false;
+  private discoveryInProgress = false;
+
+  constructor(private readonly name: string) {
+    super();
+  }
+
+  /**
+   * Force the discovery process to run. Normally invoked by the derived class when a relevant file
+   * system change is detected.
+   */
+  public refresh(): void {
+    // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
+    // got a storm of refresh requests due to, say, the copying or deletion of a large directory
+    // tree, we could potentially spawn a separate simultaneous discovery operation for each
+    // individual file change notification.
+    // Our approach is to spawn a discovery operation immediately upon receiving the first refresh
+    // request. If we receive any additional refresh requests before the first one is complete, we
+    // record this fact by setting `this.retry = true`. When the original discovery operation
+    // completes, we discard its results and spawn another one to account for that additional
+    // changes that have happened since.
+    // The means that for the common case of a single file being modified, we'll complete the
+    // discovery and update as soon as possible. If multiple files are being modified, we'll
+    // probably wind up doing discovery at least twice.
+    // We could choose to delay the initial discovery request by a second or two to wait for any
+    // other change notifications that might be coming along. However, this would create more
+    // latency in the common case, in order to save a bit of latency in the uncommon case.
+
+    if (this.discoveryInProgress) {
+      // There's already a discovery operation in progress. Tell it to restart when it's done.
+      this.retry = true;
+    }
+    else {
+      // No discovery in progress, so start one now.
+      this.discoveryInProgress = true;
+      this.launchDiscovery();
+    }
+  }
+
+  /**
+   * Starts the asynchronous discovery operation by invoking the `discover` function. When the
+   * discovery operation completes, the `update` function will be invoked with the results of the
+   * discovery.
+   */
+  private launchDiscovery(): void {
+    const discoveryPromise = this.discover();
+    discoveryPromise.then(results => {
+      if (!this.retry) {
+        // Update any listeners with the results of the discovery.
+        this.discoveryInProgress = false;
+        this.update(results);
+      }
+    });
+
+    discoveryPromise.catch(err => {
+      showAndLogErrorMessage(`${this.name} failed. Reason: ${err.message}`);
+    });
+
+    discoveryPromise.finally(() => {
+      if (this.retry) {
+        // Another refresh request came in while we were still running a previous discovery
+        // operation. Since the discovery results we just computed are now stale, we'll launch
+        // another discovery operation instead of updating.
+        // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+        // initial discovery operation failed.
+        this.retry = false;
+        this.launchDiscovery();
+      }
+    });
+  }
+
+  /**
+   * Overridden by the derived class to spawn the actual discovery operation, returning the results.
+   */
+  protected abstract discover(): Promise<T>;
+
+  /**
+   * Overridden by the derived class to atomically update the `Discovery` object with the results of
+   * the discovery operation, and to notify any listeners that the discovery results may have
+   * changed.
+   * @param results The discovery results returned by the `discover` function.
+   */
+  protected abstract update(results: T): void;
+}

extensions/ql-vscode/src/distribution.ts

@@ -1,14 +1,16 @@
-import * as fetch from "node-fetch";
-import * as fs from "fs-extra";
-import * as os from "os";
-import * as path from "path";
-import * as unzipper from "unzipper";
-import * as url from "url";
-import { ExtensionContext, Event } from "vscode";
-import { DistributionConfig } from "./config";
-import { ProgressUpdate, showAndLogErrorMessage } from "./helpers";
-import { logger } from "./logging";
-import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-version";
+import * as fetch from 'node-fetch';
+import * as fs from 'fs-extra';
+import * as os from 'os';
+import * as path from 'path';
+import * as semver from 'semver';
+import * as unzipper from 'unzipper';
+import * as url from 'url';
+import { ExtensionContext, Event } from 'vscode';
+import { DistributionConfig } from './config';
+import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from './helpers';
+import { logger } from './logging';
+import * as helpers from './helpers';
+import { getCodeQlCliVersion } from './cli-version';
 
 /**
  * distribution.ts
@@ -19,104 +21,152 @@ import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-versi
 
 /**
  * Default value for the owner name of the extension-managed distribution on GitHub.
- * 
+ *
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
-const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";
+const DEFAULT_DISTRIBUTION_OWNER_NAME = 'github';
 
 /**
  * Default value for the repository name of the extension-managed distribution on GitHub.
- * 
+ *
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
-const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-binaries";
+const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = 'codeql-cli-binaries';
 
 /**
- * Version constraint for the CLI.
- * 
+ * Range of versions of the CLI that are compatible with the extension.
+ *
  * This applies to both extension-managed and CLI distributions.
  */
-export const DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT: VersionConstraint = {
-  description: "2.0.*",
-  isVersionCompatible: (v: Version) => {
-    return v.majorVersion === 2 && v.minorVersion === 0
-  }
-}
+export const DEFAULT_DISTRIBUTION_VERSION_RANGE: semver.Range = new semver.Range('2.x');
 
 export interface DistributionProvider {
-  getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>,
-  onDidChangeDistribution?: Event<void>
+  getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
+  onDidChangeDistribution?: Event<void>;
+  getDistribution(): Promise<FindDistributionResult>;
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
+  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
     this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionConstraint);
-    this._onDidChangeDistribution = config.onDidChangeDistributionConfiguration;
-    this._versionConstraint = versionConstraint;
+    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
+    this._onDidChangeDistribution = config.onDidChangeConfiguration;
+    this._updateCheckRateLimiter = new InvocationRateLimiter(
+      extensionContext,
+      'extensionSpecificDistributionUpdateCheck',
+      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+    );
+    this._versionRange = versionRange;
   }
 
   /**
    * Look up a CodeQL launcher binary.
    */
   public async getDistribution(): Promise<FindDistributionResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath === undefined) {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    if (distribution === undefined) {
       return {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(codeQlPath, logger);
-    if (version !== undefined && !this._versionConstraint.isVersionCompatible(version)) {
+    const version = await getCodeQlCliVersion(distribution.codeQlPath, logger);
+    if (version === undefined) {
       return {
-        codeQlPath,
+        distribution,
+        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
+      };
+    }
+
+    /**
+     * Specifies whether prerelease versions of the CodeQL CLI should be accepted.
+     *
+     * Suppose a user sets the includePrerelease config option, obtains a prerelease, then decides
+     * they no longer want a prerelease, so unsets the includePrerelease config option.
+     * Unsetting the includePrerelease config option should trigger an update check, and this
+     * update check should present them an update that returns them back to a non-prerelease
+     * version.
+     *
+     * Therefore, we adopt the following:
+     *
+     * - If the user is managing their own CLI, they can use a prerelease without specifying the
+     * includePrerelease option.
+     * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
+     * includePrerelease config option is set.
+     */
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+
+    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+      return {
+        distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
         version,
       };
     }
-    if (version === undefined) {
-      return {
-        codeQlPath,
-        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
-      }
-    }
     return {
-      codeQlPath,
+      distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
       version
     };
   }
 
+  public async hasDistribution(): Promise<boolean> {
+    const result = await this.getDistribution();
+    return result.kind !== FindDistributionResultKind.NoDistribution;
+  }
+
+  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    return distribution?.codeQlPath;
+  }
+
   /**
    * Returns the path to a possibly-compatible CodeQL launcher binary, or undefined if a binary not be found.
    */
-  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+  async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath !== undefined) {
+    if (this._config.customCodeQlPath) {
       if (!await fs.pathExists(this._config.customCodeQlPath)) {
         showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
-          "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
-          "that a CodeQL executable exists at the specified path or remove the setting.");
+          'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
+          'that a CodeQL executable exists at the specified path or remove the setting.');
         return undefined;
       }
-      return this._config.customCodeQlPath;
+
+      // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
+      if (
+        deprecatedCodeQlLauncherName() &&
+        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        await this.hasNewLauncherName()
+      ) {
+        warnDeprecatedLauncher();
+      }
+      return {
+        codeQlPath: this._config.customCodeQlPath,
+        kind: DistributionKind.CustomPathConfig
+      };
     }
 
     const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
-      return extensionSpecificCodeQlPath;
+      return {
+        codeQlPath: extensionSpecificCodeQlPath,
+        kind: DistributionKind.ExtensionManaged
+      };
     }
 
     if (process.env.PATH) {
       for (const searchDirectory of process.env.PATH.split(path.delimiter)) {
-        const expectedLauncherPath = path.join(searchDirectory, codeQlLauncherName());
-        if (await fs.pathExists(expectedLauncherPath)) {
-          return expectedLauncherPath;
+        const expectedLauncherPath = await getExecutableFromDirectory(searchDirectory);
+        if (expectedLauncherPath) {
+          return {
+            codeQlPath: expectedLauncherPath,
+            kind: DistributionKind.PathEnvironmentVariable
+          };
         }
       }
-      logger.log("INFO: Could not find CodeQL on path.");
+      logger.log('INFO: Could not find CodeQL on path.');
     }
 
     return undefined;
@@ -125,26 +175,33 @@ export class DistributionManager implements DistributionProvider {
   /**
    * Check for updates to the extension-managed distribution.  If one has not already been installed,
    * this will return an update available result with the latest available release.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
-  public async checkForUpdatesToExtensionManagedDistribution(): Promise<DistributionUpdateCheckResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
+  public async checkForUpdatesToExtensionManagedDistribution(
+    minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
+    const distribution = await this.getDistributionWithoutVersionCheck();
     const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath !== undefined && codeQlPath !== extensionManagedCodeQlPath) {
+    if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
-      return createInvalidDistributionLocationResult();
+      return createInvalidLocationResult();
+    }
+    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    switch (updateCheckResult.kind) {
+      case InvocationRateLimiterResultKind.Invoked:
+        return updateCheckResult.result;
+      case InvocationRateLimiterResultKind.RateLimited:
+        return createAlreadyCheckedRecentlyResult();
     }
-    return this._extensionSpecificDistributionManager.checkForUpdatesToDistribution();
   }
 
   /**
    * Installs a release of the extension-managed distribution.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: (p: ProgressUpdate) => void): Promise<void> {
+    progressCallback?: helpers.ProgressCallback): Promise<void> {
     return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
   }
 
@@ -152,32 +209,47 @@ export class DistributionManager implements DistributionProvider {
     return this._onDidChangeDistribution;
   }
 
+  /**
+   * @return true if the non-deprecated launcher name exists on the file system
+   * in the same directory as the specified launcher only if using an external
+   * installation. False otherwise.
+   */
+  private async hasNewLauncherName(): Promise<boolean> {
+    if (!this._config.customCodeQlPath) {
+      // not managed externally
+      return false;
+    }
+    const dir = path.dirname(this._config.customCodeQlPath);
+    const newLaunderPath = path.join(dir, codeQlLauncherName());
+    return await fs.pathExists(newLaunderPath);
+  }
+
   private readonly _config: DistributionConfig;
   private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionConstraint: VersionConstraint;
+  private readonly _versionRange: semver.Range;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
+  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
     this._extensionContext = extensionContext;
     this._config = config;
-    this._versionConstraint = versionConstraint;
+    this._versionRange = versionRange;
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
     if (this.getInstalledRelease() !== undefined) {
       // An extension specific distribution has been installed.
-      const expectedLauncherPath = path.join(this.getDistributionRootPath(), codeQlLauncherName());
-      if (await fs.pathExists(expectedLauncherPath)) {
+      const expectedLauncherPath = await getExecutableFromDirectory(this.getDistributionRootPath(), true);
+      if (expectedLauncherPath) {
         return expectedLauncherPath;
       }
-      logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
-        "Will try PATH.");
+
       try {
         await this.removeDistribution();
       } catch (e) {
-        logger.log("WARNING: Tried to remove corrupted CodeQL CLI at " +
+        logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
           `${this.getDistributionStoragePath()} but encountered an error: ${e}.`);
       }
     }
@@ -187,7 +259,7 @@ class ExtensionSpecificDistributionManager {
   /**
    * Check for updates to the extension-managed distribution.  If one has not already been installed,
    * this will return an update available result with the latest available release.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async checkForUpdatesToDistribution(): Promise<DistributionUpdateCheckResult> {
@@ -195,26 +267,30 @@ class ExtensionSpecificDistributionManager {
     const extensionSpecificRelease = this.getInstalledRelease();
     const latestRelease = await this.getLatestRelease();
 
-    if (extensionSpecificRelease !== undefined && codeQlPath !== undefined && latestRelease.id === extensionSpecificRelease.id) {
-      return createDistributionAlreadyUpToDateResult();
+    if (
+      extensionSpecificRelease !== undefined &&
+      codeQlPath !== undefined &&
+      latestRelease.id === extensionSpecificRelease.id
+    ) {
+      return createAlreadyUpToDateResult();
     }
     return createUpdateAvailableResult(latestRelease);
   }
 
   /**
    * Installs a release of the extension-managed distribution.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async installDistributionRelease(release: Release,
-    progressCallback?: (p: ProgressUpdate) => void): Promise<void> {
+    progressCallback?: helpers.ProgressCallback): Promise<void> {
     await this.downloadDistribution(release, progressCallback);
     // Store the installed release within the global extension state.
     this.storeInstalledRelease(release);
   }
 
   private async downloadDistribution(release: Release,
-    progressCallback?: (p: ProgressUpdate) => void): Promise<void> {
+    progressCallback?: helpers.ProgressCallback): Promise<void> {
     try {
       await this.removeDistribution();
     } catch (e) {
@@ -222,20 +298,31 @@ class ExtensionSpecificDistributionManager {
         `but encountered an error: ${e}.`);
     }
 
-    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(release.assets[0]);
-    const tmpDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-codeql"));
+    // Filter assets to the unique one that we require.
+    const requiredAssetName = this.getRequiredAssetName();
+    const assets = release.assets.filter(asset => asset.name === requiredAssetName);
+    if (assets.length === 0) {
+      throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
+    }
+    if (assets.length > 1) {
+      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+        assets.map(asset => asset.name).join(', '));
+    }
+
+    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(assets[0]);
+    const tmpDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'vscode-codeql'));
 
     try {
-      const archivePath = path.join(tmpDirectory, "distributionDownload.zip");
+      const archivePath = path.join(tmpDirectory, 'distributionDownload.zip');
       const archiveFile = fs.createWriteStream(archivePath);
 
-      const contentLength = assetStream.headers.get("content-length");
+      const contentLength = assetStream.headers.get('content-length');
       let numBytesDownloaded = 0;
 
       if (progressCallback && contentLength !== null) {
         const totalNumBytes = parseInt(contentLength, 10);
-        const bytesToDisplayMB = (numBytes: number) => `${(numBytes/(1024*1024)).toFixed(1)} MB`;
-        const updateProgress = () => {
+        const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+        const updateProgress = (): void => {
           progressCallback({
             step: numBytesDownloaded,
             maxStep: totalNumBytes,
@@ -246,7 +333,7 @@ class ExtensionSpecificDistributionManager {
         // Display the progress straight away rather than waiting for the first chunk.
         updateProgress();
 
-        assetStream.body.on("data", data => {
+        assetStream.body.on('data', data => {
           numBytesDownloaded += data.length;
           updateProgress();
         });
@@ -254,11 +341,11 @@ class ExtensionSpecificDistributionManager {
 
       await new Promise((resolve, reject) =>
         assetStream.body.pipe(archiveFile)
-          .on("finish", resolve)
-          .on("error", reject)
+          .on('finish', resolve)
+          .on('error', reject)
       );
 
-      this.bumpDistributionFolderIndex();
+      await this.bumpDistributionFolderIndex();
 
       logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
       await extractZipArchive(archivePath, this.getDistributionStoragePath());
@@ -269,7 +356,7 @@ class ExtensionSpecificDistributionManager {
 
   /**
    * Remove the extension-managed distribution.
-   * 
+   *
    * This should not be called for a distribution that is currently in use, as remove may fail.
    */
   private async removeDistribution(): Promise<void> {
@@ -279,12 +366,36 @@ class ExtensionSpecificDistributionManager {
     }
   }
 
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  private getRequiredAssetName(): string {
+    if (os.platform() === 'linux') return 'codeql-linux64.zip';
+    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
+    if (os.platform() === 'win32') return 'codeql-win64.zip';
+    return 'codeql.zip';
+  }
+
   private async getLatestRelease(): Promise<Release> {
-    const release = await this.createReleasesApiConsumer().getLatestRelease(this._versionConstraint, this._config.includePrerelease);
-    if (release.assets.length !== 1) {
-      throw new Error("Release had an unexpected number of assets");
-    }
-    return release;
+    const requiredAssetName = this.getRequiredAssetName();
+    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    return this.createReleasesApiConsumer().getLatestRelease(
+      this._versionRange,
+      this._config.includePrerelease,
+      release => {
+        const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
+        if (matchingAssets.length === 0) {
+          // For example, this could be a release with no platform-specific assets.
+          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          return false;
+        }
+        if (matchingAssets.length > 1) {
+          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          return false;
+        }
+        return true;
+      }
+    );
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
@@ -293,17 +404,17 @@ class ExtensionSpecificDistributionManager {
     return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
   }
 
-  private bumpDistributionFolderIndex(): void {
+  private async bumpDistributionFolderIndex(): Promise<void> {
     const index = this._extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    this._extensionContext.globalState.update(
+    await this._extensionContext.globalState.update(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
   }
 
   private getDistributionStoragePath(): string {
     // Use an empty string for the initial distribution for backwards compatibility.
     const distributionFolderIndex = this._extensionContext.globalState.get(
-      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || "";
+      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || '';
     return path.join(this._extensionContext.globalStoragePath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName + distributionFolderIndex);
   }
@@ -317,34 +428,34 @@ class ExtensionSpecificDistributionManager {
     return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
   }
 
-  private storeInstalledRelease(release: Release | undefined): void {
-    this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+  private async storeInstalledRelease(release: Release | undefined): Promise<void> {
+    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
   }
 
   private readonly _config: DistributionConfig;
   private readonly _extensionContext: ExtensionContext;
-  private readonly _versionConstraint: VersionConstraint;
+  private readonly _versionRange: semver.Range;
 
-  private static readonly _currentDistributionFolderBaseName = "distribution";
-  private static readonly _currentDistributionFolderIndexStateKey = "distributionFolderIndex";
-  private static readonly _installedReleaseStateKey = "distributionRelease";
-  private static readonly _codeQlExtractedFolderName = "codeql";
+  private static readonly _currentDistributionFolderBaseName = 'distribution';
+  private static readonly _currentDistributionFolderIndexStateKey = 'distributionFolderIndex';
+  private static readonly _installedReleaseStateKey = 'distributionRelease';
+  private static readonly _codeQlExtractedFolderName = 'codeql';
 }
 
 export class ReleasesApiConsumer {
   constructor(ownerName: string, repoName: string, personalAccessToken?: string) {
     // Specify version of the GitHub API
-    this._defaultHeaders["accept"] = "application/vnd.github.v3+json";
+    this._defaultHeaders['accept'] = 'application/vnd.github.v3+json';
 
     if (personalAccessToken) {
-      this._defaultHeaders["authorization"] = `token ${personalAccessToken}`;
+      this._defaultHeaders['authorization'] = `token ${personalAccessToken}`;
     }
 
     this._ownerName = ownerName;
     this._repoName = repoName;
   }
 
-  public async getLatestRelease(versionConstraint: VersionConstraint, includePrerelease: boolean = false): Promise<Release> {
+  public async getLatestRelease(versionRange: semver.Range, includePrerelease = false, additionalCompatibilityCheck?: (release: GithubRelease) => boolean): Promise<Release> {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases`;
     const allReleases: GithubRelease[] = await (await this.makeApiCall(apiPath)).json();
     const compatibleReleases = allReleases.filter(release => {
@@ -352,24 +463,24 @@ export class ReleasesApiConsumer {
         return false;
       }
 
-      const version = tryParseVersionString(release.tag_name);
-      if (version === undefined || !versionConstraint.isVersionCompatible(version)) {
+      const version = semver.parse(release.tag_name);
+      if (version === null || !semver.satisfies(version, versionRange, { includePrerelease })) {
         return false;
       }
 
-      return true;
+      return !additionalCompatibilityCheck || additionalCompatibilityCheck(release);
     });
-    // tryParseVersionString must succeed due to the previous filtering step
+    // Tag names must all be parsable to semvers due to the previous filtering step.
     const latestRelease = compatibleReleases.sort((a, b) => {
-      const versionComparison = versionCompare(tryParseVersionString(b.tag_name)!, tryParseVersionString(a.tag_name)!);
-      if (versionComparison === 0) {
-        return b.created_at.localeCompare(a.created_at);
+      const versionComparison = semver.compare(semver.parse(b.tag_name)!, semver.parse(a.tag_name)!);
+      if (versionComparison !== 0) {
+        return versionComparison;
       }
-      return versionComparison;
+      return b.created_at.localeCompare(a.created_at, 'en-US');
     })[0];
     if (latestRelease === undefined) {
-      throw new Error("No compatible CodeQL CLI releases were found. " +
-        "Please check that the CodeQL extension is up to date.");
+      throw new Error('No compatible CodeQL CLI releases were found. ' +
+        'Please check that the CodeQL extension is up to date.');
     }
     const assets: ReleaseAsset[] = latestRelease.assets.map(asset => {
       return {
@@ -391,7 +502,7 @@ export class ReleasesApiConsumer {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases/assets/${asset.id}`;
 
     return await this.makeApiCall(apiPath, {
-      "accept": "application/octet-stream"
+      'accept': 'application/octet-stream'
     });
   }
 
@@ -401,7 +512,7 @@ export class ReleasesApiConsumer {
 
     if (!response.ok) {
       // Check for rate limiting
-      const rateLimitResetValue = response.headers.get("X-RateLimit-Reset");
+      const rateLimitResetValue = response.headers.get('X-RateLimit-Reset');
       if (response.status === 403 && rateLimitResetValue) {
         const secondsToMillisecondsFactor = 1000;
         const rateLimitResetDate = new Date(parseInt(rateLimitResetValue, 10) * secondsToMillisecondsFactor);
@@ -415,26 +526,26 @@ export class ReleasesApiConsumer {
   private async makeRawRequest(
     requestUrl: string,
     headers: { [key: string]: string },
-    redirectCount: number = 0): Promise<fetch.Response> {
+    redirectCount = 0): Promise<fetch.Response> {
     const response = await fetch.default(requestUrl, {
       headers,
-      redirect: "manual"
+      redirect: 'manual'
     });
 
-    const redirectUrl = response.headers.get("location");
+    const redirectUrl = response.headers.get('location');
     if (isRedirectStatusCode(response.status) && redirectUrl && redirectCount < ReleasesApiConsumer._maxRedirects) {
       const parsedRedirectUrl = url.parse(redirectUrl);
-      if (parsedRedirectUrl.protocol != "https:") {
-        throw new Error("Encountered a non-https redirect, rejecting");
+      if (parsedRedirectUrl.protocol != 'https:') {
+        throw new Error('Encountered a non-https redirect, rejecting');
       }
-      if (parsedRedirectUrl.host != "api.github.com") {
+      if (parsedRedirectUrl.host != 'api.github.com') {
         // Remove authorization header if we are redirected outside of the GitHub API.
         //
         // This is necessary to stream release assets since AWS fails if more than one auth
         // mechanism is provided.
-        delete headers["authorization"];
+        delete headers['authorization'];
       }
-      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1)
+      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
     }
 
     return response;
@@ -444,7 +555,7 @@ export class ReleasesApiConsumer {
   private readonly _ownerName: string;
   private readonly _repoName: string;
 
-  private static readonly _apiBase = "https://api.github.com";
+  private static readonly _apiBase = 'https://api.github.com';
   private static readonly _maxRedirects = 20;
 }
 
@@ -465,31 +576,12 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-/**
- * Comparison of semantic versions.
- * 
- * Returns a positive number if a is greater than b.
- * Returns 0 if a equals b.
- * Returns a negative number if a is less than b.
- */
-export function versionCompare(a: Version, b: Version): number {
-  if (a.majorVersion !== b.majorVersion) {
-    return a.majorVersion - b.majorVersion;
-  }
-  if (a.minorVersion !== b.minorVersion) {
-    return a.minorVersion - b.minorVersion;
-  }
-  if (a.patchVersion !== b.patchVersion) {
-    return a.patchVersion - b.patchVersion;
-  }
-  if (a.prereleaseVersion !== undefined && b.prereleaseVersion !== undefined) {
-    return a.prereleaseVersion.localeCompare(b.prereleaseVersion);
-  }
-  return 0;
+function codeQlLauncherName(): string {
+  return (os.platform() === 'win32') ? 'codeql.exe' : 'codeql';
 }
 
-function codeQlLauncherName(): string {
-  return (os.platform() === "win32") ? "codeql.cmd" : "codeql";
+function deprecatedCodeQlLauncherName(): string | undefined {
+  return (os.platform() === 'win32') ? 'codeql.cmd' : undefined;
 }
 
 function isRedirectStatusCode(statusCode: number): boolean {
@@ -500,6 +592,17 @@ function isRedirectStatusCode(statusCode: number): boolean {
  * Types and helper functions relating to those types.
  */
 
+export enum DistributionKind {
+  CustomPathConfig,
+  ExtensionManaged,
+  PathEnvironmentVariable
+}
+
+export interface Distribution {
+  codeQlPath: string;
+  kind: DistributionKind;
+}
+
 export enum FindDistributionResultKind {
   CompatibleDistribution,
   UnknownCompatibilityDistribution,
@@ -507,24 +610,33 @@ export enum FindDistributionResultKind {
   NoDistribution
 }
 
-export type FindDistributionResult = CompatibleDistributionResult | UnknownCompatibilityDistributionResult |
-  IncompatibleDistributionResult | NoDistributionResult;
+export type FindDistributionResult =
+  | CompatibleDistributionResult
+  | UnknownCompatibilityDistributionResult
+  | IncompatibleDistributionResult
+  | NoDistributionResult;
 
-interface CompatibleDistributionResult {
-  codeQlPath: string;
-  kind: FindDistributionResultKind.CompatibleDistribution;
-  version: Version
+/**
+ * A result representing a distribution of the CodeQL CLI that may or may not be compatible with
+ * the extension.
+ */
+interface DistributionResult {
+  distribution: Distribution;
+  kind: FindDistributionResultKind;
 }
 
-interface UnknownCompatibilityDistributionResult {
-  codeQlPath: string;
+interface CompatibleDistributionResult extends DistributionResult {
+  kind: FindDistributionResultKind.CompatibleDistribution;
+  version: semver.SemVer;
+}
+
+interface UnknownCompatibilityDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.UnknownCompatibilityDistribution;
 }
 
-interface IncompatibleDistributionResult {
-  codeQlPath: string;
+interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: Version;
+  version: semver.SemVer;
 }
 
 interface NoDistributionResult {
@@ -532,23 +644,31 @@ interface NoDistributionResult {
 }
 
 export enum DistributionUpdateCheckResultKind {
+  AlreadyCheckedRecentlyResult,
   AlreadyUpToDate,
-  InvalidDistributionLocation,
+  InvalidLocation,
   UpdateAvailable
 }
 
-type DistributionUpdateCheckResult = DistributionAlreadyUpToDateResult | InvalidDistributionLocationResult |
-  UpdateAvailableResult;
+type DistributionUpdateCheckResult =
+  | AlreadyCheckedRecentlyResult
+  | AlreadyUpToDateResult
+  | InvalidLocationResult
+  | UpdateAvailableResult;
 
-export interface DistributionAlreadyUpToDateResult {
+export interface AlreadyCheckedRecentlyResult {
+  kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
+}
+
+export interface AlreadyUpToDateResult {
   kind: DistributionUpdateCheckResultKind.AlreadyUpToDate;
 }
 
 /**
  * The distribution could not be installed or updated because it is not managed by the extension.
  */
-export interface InvalidDistributionLocationResult {
-  kind: DistributionUpdateCheckResultKind.InvalidDistributionLocation;
+export interface InvalidLocationResult {
+  kind: DistributionUpdateCheckResultKind.InvalidLocation;
 }
 
 export interface UpdateAvailableResult {
@@ -556,15 +676,21 @@ export interface UpdateAvailableResult {
   updatedRelease: Release;
 }
 
-function createDistributionAlreadyUpToDateResult(): DistributionAlreadyUpToDateResult {
+function createAlreadyCheckedRecentlyResult(): AlreadyCheckedRecentlyResult {
+  return {
+    kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult
+  };
+}
+
+function createAlreadyUpToDateResult(): AlreadyUpToDateResult {
   return {
     kind: DistributionUpdateCheckResultKind.AlreadyUpToDate
   };
 }
 
-function createInvalidDistributionLocationResult(): InvalidDistributionLocationResult {
+function createInvalidLocationResult(): InvalidLocationResult {
   return {
-    kind: DistributionUpdateCheckResultKind.InvalidDistributionLocation
+    kind: DistributionUpdateCheckResultKind.InvalidLocation
   };
 }
 
@@ -575,6 +701,31 @@ function createUpdateAvailableResult(updatedRelease: Release): UpdateAvailableRe
   };
 }
 
+// Exported for testing
+export async function getExecutableFromDirectory(directory: string, warnWhenNotFound = false): Promise<string | undefined> {
+  const expectedLauncherPath = path.join(directory, codeQlLauncherName());
+  const deprecatedLauncherName = deprecatedCodeQlLauncherName();
+  const alternateExpectedLauncherPath = deprecatedLauncherName ? path.join(directory, deprecatedLauncherName) : undefined;
+  if (await fs.pathExists(expectedLauncherPath)) {
+    return expectedLauncherPath;
+  } else if (alternateExpectedLauncherPath && (await fs.pathExists(alternateExpectedLauncherPath))) {
+    warnDeprecatedLauncher();
+    return alternateExpectedLauncherPath;
+  }
+  if (warnWhenNotFound) {
+    logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+      'Will try PATH.');
+  }
+  return undefined;
+}
+
+function warnDeprecatedLauncher() {
+  helpers.showAndLogWarningMessage(
+    `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
+    `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
+  );
+}
+
 /**
  * A release on GitHub.
  */
@@ -625,7 +776,7 @@ export interface GithubRelease {
   assets: GithubReleaseAsset[];
 
   /**
-   * The creation date of the release on GitHub.
+   * The creation date of the release on GitHub, in ISO 8601 format.
    */
   created_at: string;
 
@@ -670,11 +821,6 @@ export interface GithubReleaseAsset {
   size: number;
 }
 
-interface VersionConstraint {
-  description: string;
-  isVersionCompatible(version: Version): boolean;
-}
-
 export class GithubApiError extends Error {
   constructor(public status: number, public body: string) {
     super(`API call failed with status code ${status}, body: ${body}`);

extensions/ql-vscode/src/extension.ts

@@ -1,20 +1,64 @@
-import { commands, Disposable, ExtensionContext, extensions, ProgressLocation, ProgressOptions, window as Window, Uri } from 'vscode';
-import { ErrorCodes, LanguageClient, ResponseError } from 'vscode-languageclient';
+import {
+  CancellationToken,
+  commands,
+  Disposable,
+  ExtensionContext,
+  extensions,
+  languages,
+  ProgressLocation,
+  ProgressOptions,
+  Uri,
+  window as Window,
+  env,
+  window
+} from 'vscode';
+import { LanguageClient } from 'vscode-languageclient';
+import * as path from 'path';
+import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
+
+import { AstViewer } from './astViewer';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { DistributionConfigListener, QueryServerConfigListener, QueryHistoryConfigListener } from './config';
+import { CodeQLCliServer } from './cli';
+import {
+  CliConfigListener,
+  DistributionConfigListener,
+  MAX_QUERIES,
+  QueryHistoryConfigListener,
+  QueryServerConfigListener
+} from './config';
+import * as languageSupport from './languageSupport';
 import { DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
-import { DistributionUpdateCheckResultKind, DistributionManager, FindDistributionResult, FindDistributionResultKind, GithubApiError,
-  DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT, GithubRateLimitedError } from './distribution';
+import {
+  TemplateQueryDefinitionProvider,
+  TemplateQueryReferenceProvider,
+  TemplatePrintAstProvider
+} from './contextual/templateProvider';
+import {
+  DEFAULT_DISTRIBUTION_VERSION_RANGE,
+  DistributionKind,
+  DistributionManager,
+  DistributionUpdateCheckResultKind,
+  FindDistributionResult,
+  FindDistributionResultKind,
+  GithubApiError,
+  GithubRateLimitedError
+} from './distribution';
 import * as helpers from './helpers';
+import { assertNever } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
-import { InterfaceManager, WebviewReveal } from './interface';
+import { InterfaceManager } from './interface';
+import { WebviewReveal } from './interface-utils';
 import { ideServerLogger, logger, queryServerLogger } from './logging';
-import { compileAndRunQueryAgainstDatabase, EvaluationInfo, tmpDirDisposal, UserCancellationException } from './queries';
 import { QueryHistoryManager } from './query-history';
+import { CompletedQuery } from './query-results';
 import * as qsClient from './queryserver-client';
-import { CodeQLCliServer } from './cli';
-import { assertNever } from './helpers-pure';
+import { displayQuickQuery } from './quick-query';
+import { compileAndRunQueryAgainstDatabase, tmpDirDisposal } from './run-queries';
+import { QLTestAdapterFactory } from './test-adapter';
+import { TestUIService } from './test-ui';
+import { CompareInterfaceManager } from './compare/compare-interface';
+import { gatherQlFiles } from './pure/files';
 
 /**
  * extension.ts
@@ -49,76 +93,87 @@ let isInstallingOrUpdatingDistribution = false;
  *
  * @param excludedCommands List of commands for which we should not register error stubs.
  */
-function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => void) {
+function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => Promise<void>): void {
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
   const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
   const extension = extensions.getExtension(extensionId);
-  if (extension === undefined)
+  if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
+  }
 
   const stubbedCommands: string[]
     = extension.packageJSON.contributes.commands.map((entry: { command: string }) => entry.command);
 
   stubbedCommands.forEach(command => {
     if (excludedCommands.indexOf(command) === -1) {
-      errorStubs.push(commands.registerCommand(command, stubGenerator(command)));
+      errorStubs.push(helpers.commandRunner(command, stubGenerator(command)));
     }
   });
 }
 
 export async function activate(ctx: ExtensionContext): Promise<void> {
-  // Initialise logging, and ensure all loggers are disposed upon exit.
-  ctx.subscriptions.push(logger);
   logger.log('Starting CodeQL extension');
 
+  initializeLogging(ctx);
+  languageSupport.install();
+
   const distributionConfigListener = new DistributionConfigListener();
   ctx.subscriptions.push(distributionConfigListener);
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT);
+  const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
+  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
 
-  const shouldUpdateOnNextActivationKey = "shouldUpdateOnNextActivation";
+  const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
-  registerErrorStubs([checkForUpdatesCommand], command => () => {
+  registerErrorStubs([checkForUpdatesCommand], command => (async () => {
     helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
-  });
+  }));
 
-  interface ReportingConfig {
+  interface DistributionUpdateConfig {
+    isUserInitiated: boolean;
     shouldDisplayMessageWhenNoUpdates: boolean;
-    shouldErrorIfUpdateFails: boolean;
+    allowAutoUpdating: boolean;
   }
 
-  async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, reportingConfig: ReportingConfig): Promise<void> {
-    const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution();
+  async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
+    const minSecondsSinceLastUpdateCheck = config.isUserInitiated ? 0 : 86400;
+    const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
+      helpers.showAndLogInformationMessage : async (message: string) => logger.log(message);
+    const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
+
+    // We do want to auto update if there is no distribution at all
+    const allowAutoUpdating = config.allowAutoUpdating || !await distributionManager.hasDistribution();
+
     switch (result.kind) {
-      case DistributionUpdateCheckResultKind.AlreadyUpToDate:
-        if (reportingConfig.shouldDisplayMessageWhenNoUpdates) {
-          helpers.showAndLogInformationMessage("CodeQL CLI already up to date.");
-        }
+      case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
+        logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
+          `${minSecondsSinceLastUpdateCheck} seconds.`);
         break;
-      case DistributionUpdateCheckResultKind.InvalidDistributionLocation:
-        if (reportingConfig.shouldDisplayMessageWhenNoUpdates) {
-          helpers.showAndLogErrorMessage("CodeQL CLI is installed externally so could not be updated.");
-        }
+      case DistributionUpdateCheckResultKind.AlreadyUpToDate:
+        await noUpdatesLoggingFunc('CodeQL CLI already up to date.');
+        break;
+      case DistributionUpdateCheckResultKind.InvalidLocation:
+        await noUpdatesLoggingFunc('CodeQL CLI is installed externally so could not be updated.');
         break;
       case DistributionUpdateCheckResultKind.UpdateAvailable:
-        if (beganMainExtensionActivation) {
+        if (beganMainExtensionActivation || !allowAutoUpdating) {
           const updateAvailableMessage = `Version "${result.updatedRelease.name}" of the CodeQL CLI is now available. ` +
-            "The update will be installed after Visual Studio Code restarts. Restart now to upgrade?";
-          ctx.globalState.update(shouldUpdateOnNextActivationKey, true);
-          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, "Restart and Upgrade")) {
-            await commands.executeCommand("workbench.action.reloadWindow");
+            'Do you wish to upgrade?';
+          await ctx.globalState.update(shouldUpdateOnNextActivationKey, true);
+          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, 'Restart and Upgrade')) {
+            await commands.executeCommand('workbench.action.reloadWindow');
           }
         } else {
           const progressOptions: ProgressOptions = {
-            location: ProgressLocation.Notification,
             title: progressTitle,
-            cancellable: false,
+            location: ProgressLocation.Notification,
           };
+
           await helpers.withProgress(progressOptions, progress =>
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
-          ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
+          await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
           helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
         }
         break;
@@ -127,28 +182,32 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     }
   }
 
-  async function installOrUpdateDistribution(reportingConfig: ReportingConfig): Promise<void> {
+  async function installOrUpdateDistribution(config: DistributionUpdateConfig): Promise<void> {
     if (isInstallingOrUpdatingDistribution) {
-      throw new Error("Already installing or updating CodeQL CLI");
+      throw new Error('Already installing or updating CodeQL CLI');
     }
     isInstallingOrUpdatingDistribution = true;
     const codeQlInstalled = await distributionManager.getCodeQlPathWithoutVersionCheck() !== undefined;
     const willUpdateCodeQl = ctx.globalState.get(shouldUpdateOnNextActivationKey);
-    const messageText = willUpdateCodeQl ? "Updating CodeQL CLI" :
-      codeQlInstalled ? "Checking for updates to CodeQL CLI" : "Installing CodeQL CLI";
+    const messageText = willUpdateCodeQl
+      ? 'Updating CodeQL CLI'
+      : codeQlInstalled
+        ? 'Checking for updates to CodeQL CLI'
+        : 'Installing CodeQL CLI';
+
     try {
-      await installOrUpdateDistributionWithProgressTitle(messageText, reportingConfig);
+      await installOrUpdateDistributionWithProgressTitle(messageText, config);
     } catch (e) {
       // Don't rethrow the exception, because if the config is changed, we want to be able to retry installing
       // or updating the distribution.
-      const alertFunction = (codeQlInstalled && !reportingConfig.shouldErrorIfUpdateFails) ?
+      const alertFunction = (codeQlInstalled && !config.isUserInitiated) ?
         helpers.showAndLogWarningMessage : helpers.showAndLogErrorMessage;
-      const taskDescription = (willUpdateCodeQl ? "update" :
-        codeQlInstalled ? "check for updates to" : "install") + " CodeQL CLI";
+      const taskDescription = (willUpdateCodeQl ? 'update' :
+        codeQlInstalled ? 'check for updates to' : 'install') + ' CodeQL CLI';
 
       if (e instanceof GithubRateLimitedError) {
         alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
-          `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString()}.`);
+          `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString(env.language)}.`);
       } else if (e instanceof GithubApiError) {
         alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
       }
@@ -162,17 +221,31 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.rawString})`);
+        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
-      case FindDistributionResultKind.IncompatibleDistribution:
-        helpers.showAndLogWarningMessage("The current version of the CodeQL CLI is incompatible with this extension.");
+      case FindDistributionResultKind.IncompatibleDistribution: {
+        const fixGuidanceMessage = (() => {
+          switch (result.distribution.kind) {
+            case DistributionKind.ExtensionManaged:
+              return 'Please update the CodeQL CLI by running the "CodeQL: Check for CLI Updates" command.';
+            case DistributionKind.CustomPathConfig:
+              return `Please update the \"CodeQL CLI Executable Path\" setting to point to a CLI in the version range ${codeQlVersionRange}.`;
+            case DistributionKind.PathEnvironmentVariable:
+              return `Please update the CodeQL CLI on your PATH to a version compatible with ${codeQlVersionRange}, or ` +
+                `set the \"CodeQL CLI Executable Path\" setting to the path of a CLI version compatible with ${codeQlVersionRange}.`;
+          }
+        })();
+
+        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
+          'is incompatible with this extension. ' + fixGuidanceMessage);
         break;
+      }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
-        helpers.showAndLogWarningMessage("Compatibility with the configured CodeQL CLI could not be determined. " +
-          "You may experience problems using the extension.");
+        helpers.showAndLogWarningMessage('Compatibility with the configured CodeQL CLI could not be determined. ' +
+          'You may experience problems using the extension.');
         break;
       case FindDistributionResultKind.NoDistribution:
-        helpers.showAndLogErrorMessage("The CodeQL CLI could not be found.");
+        helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
         break;
       default:
         assertNever(result);
@@ -180,8 +253,8 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(reportingConfig: ReportingConfig): Promise<void> {
-    await installOrUpdateDistribution(reportingConfig);
+  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
+    await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
@@ -190,118 +263,401 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
       await activateWithInstalledDistribution(ctx, distributionManager);
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
-        const installActionName = "Install CodeQL CLI";
-        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, installActionName);
+        const installActionName = 'Install CodeQL CLI';
+        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+          items: [installActionName]
+        });
         if (chosenAction === installActionName) {
           installOrUpdateThenTryActivate({
+            isUserInitiated: true,
             shouldDisplayMessageWhenNoUpdates: false,
-            shouldErrorIfUpdateFails: true
+            allowAutoUpdating: true
           });
         }
       });
     }
   }
 
-  ctx.subscriptions.push(distributionConfigListener.onDidChangeDistributionConfiguration(() => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(distributionConfigListener.onDidChangeConfiguration(() => installOrUpdateThenTryActivate({
+    isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: false,
-    shouldErrorIfUpdateFails: true
+    allowAutoUpdating: true
   })));
-  ctx.subscriptions.push(commands.registerCommand(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(helpers.commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+    isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: true,
-    shouldErrorIfUpdateFails: true
+    allowAutoUpdating: true
   })));
 
   await installOrUpdateThenTryActivate({
+    isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
-    shouldErrorIfUpdateFails: !!ctx.globalState.get(shouldUpdateOnNextActivationKey)
+
+    // only auto update on startup if the user has previously requested an update
+    // otherwise, ask user to accept the update
+    allowAutoUpdating: !!ctx.globalState.get(shouldUpdateOnNextActivationKey)
   });
 }
 
-async function activateWithInstalledDistribution(ctx: ExtensionContext, distributionManager: DistributionManager) {
+async function activateWithInstalledDistribution(
+  ctx: ExtensionContext,
+  distributionManager: DistributionManager
+): Promise<void> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
-  errorStubs.forEach(stub => stub.dispose());
+  errorStubs.forEach((stub) => stub.dispose());
 
-  const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(distributionManager);
+  logger.log('Initializing configuration listener...');
+  const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(
+    distributionManager
+  );
   ctx.subscriptions.push(qlConfigurationListener);
 
-  ctx.subscriptions.push(queryServerLogger);
-  ctx.subscriptions.push(ideServerLogger);
-
-
-  const cliServer = new CodeQLCliServer(distributionManager, logger);
+  logger.log('Initializing CodeQL cli server...');
+  const cliServer = new CodeQLCliServer(
+    distributionManager,
+    new CliConfigListener(),
+    logger
+  );
   ctx.subscriptions.push(cliServer);
 
-  const qs = new qsClient.QueryServerClient(qlConfigurationListener, cliServer, {
-    logger: queryServerLogger,
-  }, task => Window.withProgress({ title: 'CodeQL query server', location: ProgressLocation.Window }, task));
+  logger.log('Initializing query server client.');
+  const qs = new qsClient.QueryServerClient(
+    qlConfigurationListener,
+    cliServer,
+    {
+      logger: queryServerLogger,
+    },
+    (task) =>
+      Window.withProgress(
+        { title: 'CodeQL query server', location: ProgressLocation.Window },
+        task
+      )
+  );
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
+  logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qlConfigurationListener, logger);
   ctx.subscriptions.push(dbm);
-  const databaseUI = new DatabaseUI(ctx, cliServer, dbm, qs);
+  logger.log('Initializing database panel.');
+  const databaseUI = new DatabaseUI(
+    cliServer,
+    dbm,
+    qs,
+    getContextStoragePath(ctx),
+    ctx.extensionPath
+  );
   ctx.subscriptions.push(databaseUI);
 
+  logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  const showResults = async (item: CompletedQuery) =>
+    showResultsForCompletedQuery(item, WebviewReveal.Forced);
+
   const qhm = new QueryHistoryManager(
-    ctx,
+    qs,
+    ctx.extensionPath,
     queryHistoryConfigurationListener,
-    async item => showResultsForInfo(item.info, WebviewReveal.Forced)
+    showResults,
+    async (from: CompletedQuery, to: CompletedQuery) =>
+      showResultsForComparison(from, to),
   );
+  ctx.subscriptions.push(qhm);
+  logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
+
+  logger.log('Initializing compare panel interface.');
+  const cmpm = new CompareInterfaceManager(
+    ctx,
+    dbm,
+    cliServer,
+    queryServerLogger,
+    showResults
+  );
+  ctx.subscriptions.push(cmpm);
+
+  logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
-  async function showResultsForInfo(info: EvaluationInfo, forceReveal: WebviewReveal): Promise<void> {
-    await intm.showResults(info, forceReveal, false);
+  async function showResultsForComparison(
+    from: CompletedQuery,
+    to: CompletedQuery
+  ): Promise<void> {
+    try {
+      await cmpm.showResults(from, to);
+    } catch (e) {
+      helpers.showAndLogErrorMessage(e.message);
+    }
   }
 
-  async function compileAndRunQuery(quickEval: boolean, selectedQuery: Uri | undefined) {
+  async function showResultsForCompletedQuery(
+    query: CompletedQuery,
+    forceReveal: WebviewReveal
+  ): Promise<void> {
+    await intm.showResults(query, forceReveal, false);
+  }
+
+  async function compileAndRunQuery(
+    quickEval: boolean,
+    selectedQuery: Uri | undefined,
+    progress: helpers.ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> {
     if (qs !== undefined) {
-      try {
-        const dbItem = await databaseUI.getDatabaseItem();
-        if (dbItem === undefined) {
-          throw new Error('Can\'t run query without a selected database');
-        }
-        const info = await compileAndRunQueryAgainstDatabase(cliServer, qs, dbItem, quickEval, selectedQuery);
-        await showResultsForInfo(info, WebviewReveal.NotForced);
-        qhm.push(info);
-      }
-      catch (e) {
-        if (e instanceof UserCancellationException) {
-          logger.log(e.message);
-        }
-        else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-          logger.log(e.message);
-        }
-        else if (e instanceof Error)
-          helpers.showAndLogErrorMessage(e.message);
-        else
-          throw e;
+      const dbItem = await databaseUI.getDatabaseItem(progress, token);
+      if (dbItem === undefined) {
+        throw new Error('Can\'t run query without a selected database');
       }
+      const info = await compileAndRunQueryAgainstDatabase(
+        cliServer,
+        qs,
+        dbItem,
+        quickEval,
+        selectedQuery,
+        progress,
+        token
+      );
+      const item = qhm.addQuery(info);
+      await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
+      // The call to showResults potentially creates SARIF file;
+      // Update the tree item context value to allow viewing that
+      // SARIF file from context menu.
+      await qhm.updateTreeItemContextValue(item);
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
-  let client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
-    documentSelector: [
-      { language: 'ql', scheme: 'file' },
-      { language: 'yaml', scheme: 'file', pattern: '**/qlpack.yml' }
-    ],
-    synchronize: {
-      configurationSection: 'codeQL'
+  logger.log('Initializing CodeQL language server.');
+  const client = new LanguageClient(
+    'CodeQL Language Server',
+    () => spawnIdeServer(qlConfigurationListener),
+    {
+      documentSelector: [
+        { language: 'ql', scheme: 'file' },
+        { language: 'yaml', scheme: 'file', pattern: '**/qlpack.yml' },
+      ],
+      synchronize: {
+        configurationSection: 'codeQL',
+      },
+      // Ensure that language server exceptions are logged to the same channel as its output.
+      outputChannel: ideServerLogger.outputChannel,
     },
-    // Ensure that language server exceptions are logged to the same channel as its output.
-    outputChannel: ideServerLogger.outputChannel
-  }, true);
+    true
+  );
 
-  ctx.subscriptions.push(commands.registerCommand('codeQL.runQuery', async (uri: Uri | undefined) => await compileAndRunQuery(false, uri)));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.quickEval', async (uri: Uri | undefined) => await compileAndRunQuery(true, uri)));
+  logger.log('Initializing QLTest interface.');
+  const testExplorerExtension = extensions.getExtension<TestHub>(
+    testExplorerExtensionId
+  );
+  if (testExplorerExtension) {
+    const testHub = testExplorerExtension.exports;
+    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
+    ctx.subscriptions.push(testAdapterFactory);
 
+    const testUIService = new TestUIService(testHub);
+    ctx.subscriptions.push(testUIService);
+  }
+
+  logger.log('Registering top-level command palette commands.');
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress(
+      'codeQL.runQuery',
+      async (
+        progress: helpers.ProgressCallback,
+        token: CancellationToken,
+        uri: Uri | undefined
+      ) => await compileAndRunQuery(false, uri, progress, token),
+      {
+        title: 'Running query',
+        cancellable: true
+      }
+    )
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress(
+      'codeQL.runQueries',
+      async (
+        progress: helpers.ProgressCallback,
+        token: CancellationToken,
+        _: Uri | undefined,
+        multi: Uri[]
+      ) => {
+        const maxQueryCount = MAX_QUERIES.getValue() as number;
+        const [files, dirFound] = await gatherQlFiles(multi.map(uri => uri.fsPath));
+        if (files.length > maxQueryCount) {
+          throw new Error(`You tried to run ${files.length} queries, but the maximum is ${maxQueryCount}. Try selecting fewer queries or changing the 'codeQL.runningQueries.maxQueries' setting.`);
+        }
+        // warn user and display selected files when a directory is selected because some ql
+        // files may be hidden from the user.
+        if (dirFound) {
+          const fileString = files.map(file => path.basename(file)).join(', ');
+          const res = await helpers.showBinaryChoiceDialog(
+            `You are about to run ${files.length} queries: ${fileString} Do you want to continue?`
+          );
+          if (!res) {
+            return;
+          }
+        }
+        const queryUris = files.map(path => Uri.parse(`file:${path}`, true));
+
+        // Use a wrapped progress so that messages appear with the queries remaining in it.
+        let queriesRemaining = queryUris.length;
+        function wrappedProgress(update: helpers.ProgressUpdate) {
+          const message = queriesRemaining > 1
+            ? `${queriesRemaining} remaining. ${update.message}`
+            : update.message;
+          progress({
+            ...update,
+            message
+          });
+        }
+
+        if (queryUris.length > 1) {
+          // Try to upgrade the current database before running any queries
+          // so that the user isn't confronted with multiple upgrade
+          // requests for each query to run.
+          // Only do it if running multiple queries since this check is
+          // performed on each query run anyway.
+          await databaseUI.tryUpgradeCurrentDatabase(progress, token);
+        }
+
+        wrappedProgress({
+          maxStep: queryUris.length,
+          step: queryUris.length - queriesRemaining,
+          message: ''
+        });
+
+        await Promise.all(queryUris.map(async uri =>
+          compileAndRunQuery(false, uri, wrappedProgress, token)
+            .then(() => queriesRemaining--)
+        ));
+      },
+      {
+        title: 'Running queries',
+        cancellable: true
+      })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress(
+      'codeQL.quickEval',
+      async (
+        progress: helpers.ProgressCallback,
+        token: CancellationToken,
+        uri: Uri | undefined
+      ) => await compileAndRunQuery(true, uri, progress, token),
+      {
+        title: 'Running query',
+        cancellable: true
+      })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress('codeQL.quickQuery', async (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      displayQuickQuery(ctx, cliServer, databaseUI, progress, token),
+      {
+        title: 'Run Quick Query'
+      }
+    )
+  );
+
+  ctx.subscriptions.push(
+    helpers.commandRunner('codeQL.restartQueryServer', async () => {
+      await qs.restartQueryServer();
+      helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
+        outputLogger: queryServerLogger,
+      });
+    })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunner('codeQL.chooseDatabaseFolder', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseFolder(progress, token)
+    )
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunner('codeQL.chooseDatabaseArchive', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseArchive(progress, token)
+    )
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseLgtm(progress, token),
+      {
+        title: 'Adding database from LGTM',
+      })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseInternet(progress, token),
+
+      {
+        title: 'Adding database from URL',
+      })
+  );
+
+  logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
+
+  // Jump-to-definition and find-references
+  logger.log('Registering jump-to-definition handlers.');
+  languages.registerDefinitionProvider(
+    { scheme: archiveFilesystemProvider.zipArchiveScheme },
+    new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
+  );
+
+  languages.registerReferenceProvider(
+    { scheme: archiveFilesystemProvider.zipArchiveScheme },
+    new TemplateQueryReferenceProvider(cliServer, qs, dbm)
+  );
+
+  const astViewer = new AstViewer();
+  ctx.subscriptions.push(astViewer);
+  ctx.subscriptions.push(helpers.commandRunnerWithProgress('codeQL.viewAst', async (
+    progress: helpers.ProgressCallback,
+    token: CancellationToken
+  ) => {
+    const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)
+      .provideAst(window.activeTextEditor?.document);
+    if (ast) {
+      astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
+    }
+  }, {
+    cancellable: true,
+    title: 'Calculate AST'
+  }));
+
+  logger.log('Successfully finished extension initialization.');
+}
+
+function getContextStoragePath(ctx: ExtensionContext) {
+  return ctx.storagePath || ctx.globalStoragePath;
+}
+
+function initializeLogging(ctx: ExtensionContext): void {
+  const storagePath = getContextStoragePath(ctx);
+  logger.init(storagePath);
+  queryServerLogger.init(storagePath);
+  ideServerLogger.init(storagePath);
+  ctx.subscriptions.push(logger);
+  ctx.subscriptions.push(queryServerLogger);
+  ctx.subscriptions.push(ideServerLogger);
 }
 
 const checkForUpdatesCommand = 'codeQL.checkForUpdatesToCLI';

extensions/ql-vscode/src/helpers.ts

@@ -1,7 +1,29 @@
+import * as fs from 'fs-extra';
+import * as glob from 'glob-promise';
+import * as yaml from 'js-yaml';
 import * as path from 'path';
-import { CancellationToken, ProgressOptions, window as Window, workspace } from 'vscode';
+import {
+  CancellationToken,
+  ExtensionContext,
+  ProgressOptions,
+  window as Window,
+  workspace,
+  commands,
+  Disposable,
+  ProgressLocation
+} from 'vscode';
+import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
-import { EvaluationInfo } from './queries';
+
+export class UserCancellationException extends Error {
+  /**
+   * @param message The error message
+   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
+   */
+  constructor(message?: string, public readonly silent = false) {
+    super(message);
+  }
+}
 
 export interface ProgressUpdate {
   /**
@@ -18,18 +40,59 @@ export interface ProgressUpdate {
   message: string;
 }
 
+export type ProgressCallback = (p: ProgressUpdate) => void;
+
+/**
+ * A task that handles command invocations from `commandRunner`
+ * and includes a progress monitor.
+ *
+ *
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `ProgressTask` instance.
+ *
+ * @param progress a progress handler function. Call this
+ * function with a `ProgressUpdate` instance in order to
+ * denote some progress being achieved on this task.
+ * @param token a cencellation token
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+export type ProgressTask<R> = (
+  progress: ProgressCallback,
+  token: CancellationToken,
+  ...args: any[]
+) => Thenable<R>;
+
+/**
+ * A task that handles command invocations from `commandRunner`.
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `NoProgressTask` instance.
+ *
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+type NoProgressTask = ((...args: any[]) => Promise<any>);
+
 /**
  * This mediates between the kind of progress callbacks we want to
  * write (where we *set* current progress position and give
  * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%)
+ * (which increment progress by a certain amount out of 100%).
+ *
+ * Where possible, the `commandRunner` function below should be used
+ * instead of this function. The commandRunner is meant for wrapping
+ * top-level commands and provides error handling and other support
+ * automatically.
+ *
+ * Only use this function if you need a progress monitor and the
+ * control flow does not always come from a command (eg- during
+ * extension activation, or from an internal language server
+ * request).
  */
 export function withProgress<R>(
   options: ProgressOptions,
-  task: (
-    progress: (p: ProgressUpdate) => void,
-    token: CancellationToken
-  ) => Thenable<R>
+  task: ProgressTask<R>,
+  ...args: any[]
 ): Thenable<R> {
   let progressAchieved = 0;
   return Window.withProgress(options,
@@ -39,66 +102,156 @@ export function withProgress<R>(
         const increment = 100 * (step - progressAchieved) / maxStep;
         progressAchieved = step;
         progress.report({ message, increment });
-      }, token);
+      }, token, ...args);
     });
 }
 
+/**
+ * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
+ *
+ * In this variant of the command runner, no progress monitor is used.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task.
+ */
+export function commandRunner(
+  commandId: string,
+  task: NoProgressTask,
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    try {
+      await task(...args);
+    } catch (e) {
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          logger.log(e.message);
+        } else {
+          showAndLogWarningMessage(e.message);
+        }
+      } else {
+        showAndLogErrorMessage(e.message || e);
+      }
+    }
+  });
+}
+
+/**
+ * A generic wrapper for command registration.  This wrapper adds uniform error handling,
+ * progress monitoring, and cancellation for commands.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task after the progress callback
+ * and cancellation token.
+ * @param progressOptions Progress options to be sent to the progress monitor.
+ */
+export function commandRunnerWithProgress<R>(
+  commandId: string,
+  task: ProgressTask<R>,
+  progressOptions: Partial<ProgressOptions>
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const progressOptionsWithDefaults = {
+      location: ProgressLocation.Notification,
+      ...progressOptions
+    };
+    try {
+      await withProgress(progressOptionsWithDefaults, task, ...args);
+    } catch (e) {
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          logger.log(e.message);
+        } else {
+          showAndLogWarningMessage(e.message);
+        }
+      } else {
+        showAndLogErrorMessage(e.message || e);
+      }
+    }
+  });
+}
+
 /**
  * Show an error message and log it to the console
  *
- * @param message — The message to show.
- * @param items — A set of items that will be rendered as actions in the message.
+ * @param message The message to show.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return — A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogErrorMessage(message: string, ...items: string[]): Thenable<string | undefined> {
-  logger.log(message);
-  return Window.showErrorMessage(message, ...items);
+export async function showAndLogErrorMessage(message: string, {
+  outputLogger = logger,
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
 }
 /**
  * Show a warning message and log it to the console
  *
- * @param message — The message to show.
- * @param items — A set of items that will be rendered as actions in the message.
+ * @param message The message to show.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return — A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogWarningMessage(message: string, ...items: string[]): Thenable<string | undefined> {
-  logger.log(message);
-  return Window.showWarningMessage(message, ...items);
+export async function showAndLogWarningMessage(message: string, {
+  outputLogger = logger,
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showWarningMessage);
 }
 /**
  * Show an information message and log it to the console
  *
- * @param message — The message to show.
- * @param items — A set of items that will be rendered as actions in the message.
+ * @param message The message to show.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return — A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogInformationMessage(message: string, ...items: string[]): Thenable<string | undefined> {
-  logger.log(message);
-  return Window.showInformationMessage(message, ...items);
+export async function showAndLogInformationMessage(message: string, {
+  outputLogger = logger,
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage);
+}
+
+type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
+
+async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
+  fn: ShowMessageFn): Promise<string | undefined> {
+  const label = 'Show Log';
+  outputLogger.log(message);
+  const result = await fn(message, label, ...items);
+  if (result === label) {
+    outputLogger.show();
+  }
+  return result;
 }
 
 /**
  * Opens a modal dialog for the user to make a yes/no choice.
- * @param message — The message to show.
+ * @param message The message to show.
  *
- * @return — `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
+ * @return `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
  */
 export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
   const yesItem = { title: 'Yes', isCloseAffordance: false };
-  const noItem = { title: 'No', isCloseAffordance: true }
+  const noItem = { title: 'No', isCloseAffordance: true };
   const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
-  return chosenItem === yesItem;
+  return chosenItem?.title === yesItem.title;
 }
 
 /**
  * Show an information message with a customisable action.
- * @param message — The message to show.
- * @param actionMessage - The call to action message.
+ * @param message The message to show.
+ * @param actionMessage The call to action message.
  *
- * @return — `true` if the user clicks the action, `false` if the user cancels the dialog.
+ * @return `true` if the user clicks the action, `false` if the user cancels the dialog.
  */
 export async function showInformationMessageWithAction(message: string, actionMessage: string): Promise<boolean> {
   const actionItem = { title: actionMessage, isCloseAffordance: false };
@@ -109,28 +262,205 @@ export async function showInformationMessageWithAction(message: string, actionMe
 /** Gets all active workspace folders that are on the filesystem. */
 export function getOnDiskWorkspaceFolders() {
   const workspaceFolders = workspace.workspaceFolders || [];
-  let diskWorkspaceFolders: string[] = [];
+  const diskWorkspaceFolders: string[] = [];
   for (const workspaceFolder of workspaceFolders) {
-    if (workspaceFolder.uri.scheme === "file")
-      diskWorkspaceFolders.push(workspaceFolder.uri.fsPath)
+    if (workspaceFolder.uri.scheme === 'file')
+      diskWorkspaceFolders.push(workspaceFolder.uri.fsPath);
   }
   return diskWorkspaceFolders;
 }
 
 /**
- * Gets a human-readable name for an evaluated query.
- * Uses metadata if it exists, and defaults to the query file name.
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
  */
-export function getQueryName(info: EvaluationInfo) {
-  // Queries run through quick evaluation are not usually the entire query file.
-  // Label them differently and include the line numbers.
-  if (info.query.quickEvalPosition !== undefined) {
-    const { line, endLine, fileName } = info.query.quickEvalPosition;
-    const lineInfo = line === endLine ? `${line}` : `${line}-${endLine}`;
-    return `Quick evaluation of ${path.basename(fileName)}:${lineInfo}`;
-  } else if (info.query.metadata && info.query.metadata.name) {
-    return info.query.metadata.name;
-  } else {
-    return path.basename(info.query.program.queryPath);
+export class InvocationRateLimiter<T> {
+  constructor(
+    extensionContext: ExtensionContext,
+    funcIdentifier: string,
+    func: () => Promise<T>,
+    createDate: (dateString?: string) => Date = s => s ? new Date(s) : new Date()) {
+    this._createDate = createDate;
+    this._extensionContext = extensionContext;
+    this._func = func;
+    this._funcIdentifier = funcIdentifier;
+  }
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(minSecondsSinceLastInvocation: number): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this._createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 > updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this._func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined =
+      this._extensionContext.globalState.get(InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier);
+    return maybeDateString ? this._createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this._extensionContext.globalState.update(InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier, date);
+  }
+
+  private readonly _createDate: (dateString?: string) => Date;
+  private readonly _extensionContext: ExtensionContext;
+  private readonly _func: () => Promise<T>;
+  private readonly _funcIdentifier: string;
+
+  private static readonly _invocationRateLimiterPrefix = 'invocationRateLimiter_lastInvocationDate_';
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited
+  };
+}
+
+
+export type DatasetFolderInfo = {
+  dbscheme: string;
+  qlpack: string;
+}
+
+export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
+  const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
+  const packs: { packDir: string | undefined; packName: string }[] =
+    Object.entries(qlpacks).map(([packName, dirs]) => {
+      if (dirs.length < 1) {
+        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
+        return { packName, packDir: undefined };
+      }
+      if (dirs.length > 1) {
+        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
+      }
+      return {
+        packName,
+        packDir: dirs[0]
+      };
+    });
+  for (const { packDir, packName } of packs) {
+    if (packDir !== undefined) {
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme: string };
+      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
+        return packName;
+      }
+    }
+  }
+  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+}
+
+export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise<DatasetFolderInfo> {
+  const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'));
+
+  if (dbschemes.length < 1) {
+    throw new Error(`Can't find dbscheme for current database in ${datasetFolder}`);
+  }
+
+  dbschemes.sort();
+  const dbscheme = dbschemes[0];
+  if (dbschemes.length > 1) {
+    Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
+  }
+
+  const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
+  return { dbscheme, qlpack };
+}
+
+/**
+ * A cached mapping from strings to value of type U.
+ */
+export class CachedOperation<U> {
+  private readonly operation: (t: string) => Promise<U>;
+  private readonly cached: Map<string, U>;
+  private readonly lru: string[];
+  private readonly inProgressCallbacks: Map<string, [(u: U) => void, (reason?: any) => void][]>;
+
+  constructor(operation: (t: string) => Promise<U>, private cacheSize = 100) {
+    this.operation = operation;
+    this.lru = [];
+    this.inProgressCallbacks = new Map<string, [(u: U) => void, (reason?: any) => void][]>();
+    this.cached = new Map<string, U>();
+  }
+
+  async get(t: string): Promise<U> {
+    // Try and retrieve from the cache
+    const fromCache = this.cached.get(t);
+    if (fromCache !== undefined) {
+      // Move to end of lru list
+      this.lru.push(this.lru.splice(this.lru.findIndex(v => v === t), 1)[0]);
+      return fromCache;
+    }
+    // Otherwise check if in progress
+    const inProgressCallback = this.inProgressCallbacks.get(t);
+    if (inProgressCallback !== undefined) {
+      // If so wait for it to resolve
+      return await new Promise((resolve, reject) => {
+        inProgressCallback.push([resolve, reject]);
+      });
+    }
+
+    // Otherwise compute the new value, but leave a callback to allow sharing work
+    const callbacks: [(u: U) => void, (reason?: any) => void][] = [];
+    this.inProgressCallbacks.set(t, callbacks);
+    try {
+      const result = await this.operation(t);
+      callbacks.forEach(f => f[0](result));
+      this.inProgressCallbacks.delete(t);
+      if (this.lru.length > this.cacheSize) {
+        const toRemove = this.lru.shift()!;
+        this.cached.delete(toRemove);
+      }
+      this.lru.push(t);
+      this.cached.set(t, result);
+      return result;
+    } catch (e) {
+      // Rethrow error on all callbacks
+      callbacks.forEach(f => f[1](e));
+      throw e;
+    } finally {
+      this.inProgressCallbacks.delete(t);
+    }
   }
 }