Update changelog in preparation for v1.2.2 release (#433)

* Update changelog in preparation for v1.2.2 release

Co-authored-by: Shati Patel <42641846+shati-patel@users.noreply.github.com>

.github/workflows/codeql.yml

@@ -0,0 +1,21 @@
+name: "Code Scanning - CodeQL"
+
+on:
+  push:
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  codeql:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

.github/workflows/main.yml

@@ -18,11 +18,12 @@ jobs:
         with:
           node-version: '10.18.1'
 
+      - name: Install dependencies
+        run: node common/scripts/install-run-rush.js install
+        shell: bash
+
       - name: Build
-        run: |
-          cd build
-          npm install
-          npm run build-ci
+        run: node common/scripts/install-run-rush.js build
         shell: bash
 
       - name: Prepare artifacts
@@ -55,11 +56,12 @@ jobs:
           node-version: '10.18.1'
 
       # We have to build the dependencies in `lib` before running any tests.
+      - name: Install dependencies
+        run: node common/scripts/install-run-rush.js install
+        shell: bash
+
       - name: Build
-        run: |
-          cd build
-          npm install
-          npm run build-ci
+        run: node common/scripts/install-run-rush.js build
         shell: bash
 
       - name: Lint

.github/workflows/release.yml

@@ -33,12 +33,12 @@ jobs:
         with:
           node-version: '10.18.1'
 
+      - name: Install dependencies
+        run: node common/scripts/install-run-rush.js install
+        shell: bash
+
       - name: Build
-        run: |
-          cd build
-          npm install
-          # Release build instead of dev build.
-          npm run build-release
+        run: node common/scripts/install-run-rush.js build --release
         shell: bash
 
       - name: Prepare artifacts

build/README.md

@@ -1,12 +0,0 @@
-GitHub Actions Build directory
-===
-
-The point of this directory is to allow us to do a local installation *of* the rush
-tool, since 
- - installing globally is not permitted on github actions
- - installing locally in the root directory of the repo creates `node_modules` there,
-   and rush itself gives error messages since it thinks `node_modules` is not supposed 
-   to exist, since rush is supposed to be managing subproject dependencies.
-
-Running rush from a subdirectory searches parent directories for `rush.json`
-and does the build starting from that file's location.

build/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "build",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "dependencies": {
-    "@microsoft/rush": "^5.10.3"
-  },
-  "devDependencies": {},
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "build": "rush update && rush build",
-    "build-ci": "rush install && rush build",
-    "build-release": "rush install && rush build --release"
-  },
-  "author": "GitHub"
-}

common/config/rush/pnpm-lock.yaml

@@ -26,6 +26,7 @@ dependencies:
   '@types/react': 16.9.23
   '@types/react-dom': 16.9.5
   '@types/sarif': 2.1.2
+  '@types/semver': 7.2.0
   '@types/sinon': 7.5.2
   '@types/sinon-chai': 3.2.3
   '@types/through2': 2.0.34
@@ -68,6 +69,7 @@ dependencies:
   react: 16.13.0
   react-dom: 16.13.0_react@16.13.0
   reflect-metadata: 0.1.13
+  semver: 7.3.2
   sinon: 9.0.1
   sinon-chai: 3.5.0_chai@4.2.0+sinon@9.0.1
   style-loader: 0.23.1
@@ -84,7 +86,7 @@ dependencies:
   vsce: 1.74.0
   vscode-jsonrpc: 5.0.1
   vscode-languageclient: 6.1.3
-  vscode-test: 1.3.0
+  vscode-test: 1.4.0
   vscode-test-adapter-api: 1.7.0
   vscode-test-adapter-util: 0.7.0
   webpack: 4.42.0_webpack@4.42.0
@@ -508,6 +510,12 @@ packages:
     dev: false
     resolution:
       integrity: sha512-TELZl5h48KaB6SFZqTuaMEw1hrGuusbBcH+yfMaaHdS2pwDr3RTH4CVN0LyY1kqSiDm9PPvAMx8FJ0LUZreOCQ==
+  /@types/semver/7.2.0:
+    dependencies:
+      '@types/node': 12.12.30
+    dev: false
+    resolution:
+      integrity: sha512-TbB0A8ACUWZt3Y6bQPstW9QNbhNeebdgLX4T/ZfkrswAfUzRiXrgd9seol+X379Wa589Pu4UEx9Uok0D4RjRCQ==
   /@types/sinon-chai/3.2.3:
     dependencies:
       '@types/chai': 4.2.11
@@ -6210,6 +6218,13 @@ packages:
     hasBin: true
     resolution:
       integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
+  /semver/7.3.2:
+    dev: false
+    engines:
+      node: '>=10'
+    hasBin: true
+    resolution:
+      integrity: sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
   /serialize-javascript/2.1.2:
     dev: false
     resolution:
@@ -7437,7 +7452,7 @@ packages:
       vscode: ^1.24.0
     resolution:
       integrity: sha512-eAsB8koXct5JytvUcV62wLEBCQfsoclauzMLEFT6H0qBr1h8LyRc+dGDcs48pO28yFOo6VV+5AwCRLxTKh7TzQ==
-  /vscode-test/1.3.0:
+  /vscode-test/1.4.0:
     dependencies:
       http-proxy-agent: 2.1.0
       https-proxy-agent: 2.2.4
@@ -7446,7 +7461,7 @@ packages:
     engines:
       node: '>=8.9.3'
     resolution:
-      integrity: sha512-LddukcBiSU2FVTDr3c1D8lwkiOvwlJdDL2hqVbn6gIz+rpTqUCkMZSKYm94Y1v0WXlHSDQBsXyY+tchWQgGVsw==
+      integrity: sha512-Jt7HNGvSE0+++Tvtq5wc4hiXLIr2OjDShz/gbAfM/mahQpy4rKBnmOK33D+MR67ATWviQhl+vpmU3p/qwSH/Pg==
   /watchpack/1.6.0:
     dependencies:
       chokidar: 2.1.8
@@ -7921,6 +7936,7 @@ packages:
       '@types/react': 16.9.23
       '@types/react-dom': 16.9.5
       '@types/sarif': 2.1.2
+      '@types/semver': 7.2.0
       '@types/sinon': 7.5.2
       '@types/sinon-chai': 3.2.3
       '@types/tmp': 0.1.0
@@ -7955,6 +7971,7 @@ packages:
       proxyquire: 2.1.3
       react: 16.13.0
       react-dom: 16.13.0_react@16.13.0
+      semver: 7.3.2
       sinon: 9.0.1
       sinon-chai: 3.5.0_chai@4.2.0+sinon@9.0.1
       style-loader: 0.23.1
@@ -7970,7 +7987,7 @@ packages:
       vsce: 1.74.0
       vscode-jsonrpc: 5.0.1
       vscode-languageclient: 6.1.3
-      vscode-test: 1.3.0
+      vscode-test: 1.4.0
       vscode-test-adapter-api: 1.7.0
       vscode-test-adapter-util: 0.7.0
       webpack: 4.42.0_webpack@4.42.0
@@ -7978,7 +7995,7 @@ packages:
     dev: false
     name: '@rush-temp/vscode-codeql'
     resolution:
-      integrity: sha512-YwJoYdN8GMZlZHiLXhC1jw2BfrBJOpoCDtKQ78HphTslH7S94cUbASmZCgXKPkb9aIijsOY3JHE4/Od6lqB65w==
+      integrity: sha512-bU6tGSUD6TzMa6XDiDymvfY28xtDKp6uYPVCwiy7zdsl5NYUxph5Yua0Snoam7oytdYMa2HieTn8Lh6Hkb5P/A==
       tarball: 'file:projects/vscode-codeql.tgz'
     version: 0.0.0
 registry: ''
@@ -8010,6 +8027,7 @@ specifiers:
   '@types/react': ^16.8.17
   '@types/react-dom': ^16.8.4
   '@types/sarif': ~2.1.2
+  '@types/semver': ~7.2.0
   '@types/sinon': ~7.5.2
   '@types/sinon-chai': ~3.2.3
   '@types/through2': ~2.0.34
@@ -8052,6 +8070,7 @@ specifiers:
   react: ^16.8.6
   react-dom: ^16.8.6
   reflect-metadata: ~0.1.13
+  semver: ~7.3.2
   sinon: ~9.0.0
   sinon-chai: ~3.5.0
   style-loader: ~0.23.1
@@ -8068,7 +8087,7 @@ specifiers:
   vsce: ^1.65.0
   vscode-jsonrpc: ^5.0.1
   vscode-languageclient: ^6.1.3
-  vscode-test: ^1.0.0
+  vscode-test: ^1.4.0
   vscode-test-adapter-api: ~1.7.0
   vscode-test-adapter-util: ~0.7.0
   webpack: ^4.38.0

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,27 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.2.2 - 8 June 2020
+
+- Fix auto-indentation rules.
+- Add ability to download platform-specific releases of the CodeQL CLI if they are available.
+- Fix handling of downloading prerelease versions of the CodeQL CLI.
+- Add pagination for displaying non-interpreted results.
+
+## 1.2.1 - 29 May 2020
+
+- Better formatting and autoindentation when adding QLDoc comments to `.ql` and `.qll` files.
+- Allow for more flexibility when opening a database in the workspace. A user can now choose the actual database folder, or the nested `db-*` folder.
+- Add query history menu command for viewing corresponding SARIF file.
+- Add ability for users to download databases directly from LGTM.com.
+
+## 1.2.0 - 19 May 2020
+
+- Enable 'Go to Definition' and 'Go to References' on source archive
+  files in CodeQL databases. This is handled by a CodeQL query.
+- Fix adding database archive files on Windows.
+- Enable adding remote and local database archive files from the
+  command palette.
+
 ## 1.1.5 - 15 May 2020
 
 - Links in results are no longer underlined and monospaced.

extensions/ql-vscode/README.md

@@ -2,10 +2,10 @@
 
 This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
 
-* Enables you to use CodeQL to query databases generated from source code.
-* Shows the flow of data through the results of path queries, which is essential for triaging security results.
-* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/Semmle/ql).
-* Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
+- Enables you to use CodeQL to query databases generated from source code.
+- Shows the flow of data through the results of path queries, which is essential for triaging security results.
+- Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
+- Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
 To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
 
@@ -14,18 +14,18 @@ To see what has changed in the last few versions of the extension, see the [Chan
 The information in this `README` file describes the quickest way to start using CodeQL.
 For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
 
-**Quick start: Installing and configuring the extension**
+### Quick start: Installing and configuring the extension
 
 1. [Install the extension](#installing-the-extension).
 1. [Check access to the CodeQL CLI](#checking-access-to-the-codeql-cli).
 1. [Clone the CodeQL starter workspace](#cloning-the-codeql-starter-workspace).
 
-**Quick start: Using CodeQL**
+### Quick start: Using CodeQL
 
 1. [Import a database from LGTM](#importing-a-database-from-lgtm).
 1. [Run a query](#running-a-query).
 
------
+---
 
 ## Quick start: Installing and configuring the extension
 
@@ -49,11 +49,26 @@ If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Lo
 ### Cloning the CodeQL starter workspace
 
 When you're working with CodeQL, you need access to the standard CodeQL libraries and queries.
-Initially, we recommend that you clone and use the ready-to-use starter workspace, https://github.com/github/vscode-codeql-starter/.
+Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
 For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
 
+## Upgrading CodeQL standard libraries
+
+You can easily keep up-to-date with the latest changes to the [CodeQL standard libraries](https://github.com/github/codeql).
+
+If you're using the [CodeQL starter workspace](https://github.com/github/vscode-codeql-starter/), you can pull in the latest standard libraries by running:
+
+```shell
+git pull
+git submodule update --recursive
+```
+
+in the starter workspace directory.
+
+If you're using your own clone of the CodeQL standard libraries, you can do a `git pull` from where you have the libraries checked out.
+
 ## Quick start: Using CodeQL
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.
@@ -62,16 +77,13 @@ You can find all the commands contributed by the extension in the Command Palett
 
 While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
 
-1. Log in to LGTM.com.
-1. Find a project you're interested in and display the **Integrations** tab (for example, [Apache Kafka](https://lgtm.com/projects/g/apache/kafka/ci/)).
-1. Scroll to the **CodeQL databases for local analysis** section at the bottom of the page.
-1. Download databases for the languages that you want to explore.
-1. Unzip the databases.
-1. For each database that you want to import:
-    1. In the VS Code sidebar, go to **CodeQL** > **Databases** and click **+**.
-    1. Browse to the unzipped database folder (the parent folder that contains `db-<language>` and `src`) and select **Choose database** to add it.
-
-When the import is complete, each CodeQL database is displayed in the CodeQL sidebar under **Databases**.
+1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
+1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
+1. Copy the link to that project, for example `https://lgtm.com/projects/g/apache/kafka`.
+1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from LGTM** command.
+1. Paste the link you copied earlier.
+1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
+1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
 ### Running a query
 
@@ -79,7 +91,7 @@ The instructions below assume that you're using the CodeQL starter workspace, or
 
 1. Expand the `ql` folder and locate a query to run. The standard queries are grouped by target language and then type, for example: `ql/java/ql/src/Likely Bugs`.
 1. Open a query (`.ql`) file.
-3. Right-click in the query window and select **CodeQL: Run Query**. Alternatively, open the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**), type `Run Query`, then select **CodeQL: Run Query**.
+1. Right-click in the query window and select **CodeQL: Run Query**. Alternatively, open the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**), type `Run Query`, then select **CodeQL: Run Query**.
 
 The CodeQL extension runs the query on the current database using the CLI and reports progress in the bottom right corner of the application.
 When the results are ready, they're displayed in the CodeQL Query Results view. Use the dropdown menu to choose between different forms of result output.
@@ -90,10 +102,10 @@ If there are any problems running a query, a notification is displayed in the bo
 
 For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
 
-* [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
-* [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-* [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
-* [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
+- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
+- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
 

extensions/ql-vscode/language-configuration.json

@@ -1,72 +1,34 @@
 {
-    "comments": {
-        // symbol used for single line comment. Remove this entry if your language does not support line comments
-        "lineComment": "//",
-        // symbols used for start and end a block comment. Remove this entry if your language does not support block comments
-        "blockComment": [
-            "/*",
-            "*/"
-        ]
-    },
-    // symbols used as brackets
-    "brackets": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ]
-    ],
-    // symbols that are auto closed when typing
-    "autoClosingPairs": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ],
-        [
-            "\"",
-            "\""
-        ],
-        [
-            "'",
-            "'"
-        ]
-    ],
-    // symbols that that can be used to surround a selection
-    "surroundingPairs": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ],
-        [
-            "\"",
-            "\""
-        ],
-        [
-            "'",
-            "'"
-        ]
-    ]
-}
+  "comments": {
+    "lineComment": "//",
+    "blockComment": ["/*", "*/"]
+  },
+  "brackets": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"]
+  ],
+  "autoClosingPairs": [
+    { "open": "{", "close": "}" },
+    { "open": "[", "close": "]" },
+    { "open": "(", "close": ")" },
+    { "open": "'", "close": "'", "notIn": ["string", "comment"] },
+    { "open": "\"", "close": "\"", "notIn": ["string"] },
+    { "open": "/**", "close": " */", "notIn": ["string"] }
+  ],
+  "autoCloseBefore": ";:.=}])> \n\t",
+  "surroundingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    ["'", "'"],
+    ["\"", "\""]
+  ],
+  "folding": {
+    "markers": {
+      "start": "^\\s*//\\s*#?region\\b",
+      "end": "^\\s*//\\s*#?endregion\\b"
+    }
+  },
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+}

extensions/ql-vscode/media/dark/lgtm-plus.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none">
+  <path d="M16.010 6.49c-3.885 0-7.167 0.906-9.328 2.813-0.063-0.12-0.109-0.219-0.188-0.339-0.224-0.365-0.438-0.776-1.104-1.188-0.411-0.26-0.87-0.438-1.349-0.516-0.208-0.021-0.422-0.021-0.63 0l0.135-0.016c-1.214 0-1.922 0.724-2.385 1.354-0.458 0.625-0.755 1.328-0.948 2.099-0.38 1.542-0.385 3.536 1.083 5.026 0.766 0.781 1.667 1.151 2.484 1.37 0.156 0.042 0.297 0.052 0.448 0.083 0.531 2.521 2.104 4.656 4.208 5.839v0.005c1.24 0.693 2.417 1.010 3.297 1.349 1.234 0.479 2.536 1 4.052 1.135l0.078 0.005h0.198c1.745 0 3.063-0.703 4.203-1.141 0.875-0.333 2.052-0.641 3.302-1.344 0.578-0.323 1.115-0.719 1.594-1.172 1.318-1.234 2.229-2.839 2.625-4.599 1.115-0.182 2.141-0.719 2.922-1.536 1.464-1.484 1.458-3.479 1.078-5.021-0.193-0.771-0.49-1.474-0.948-2.099-0.458-0.63-1.172-1.354-2.385-1.354l0.135 0.016c-0.208-0.021-0.422-0.021-0.63 0-0.479 0.078-0.938 0.255-1.344 0.516-0.667 0.411-0.88 0.823-1.104 1.182-0.073 0.12-0.12 0.219-0.188 0.333-2.156-1.901-5.432-2.802-9.313-2.802zM16.042 8.313c4.745 0 8.016 1.422 9.411 3.964 0.839-0.323 1.453-2.521 2.146-2.948 0.563-0.344 0.885-0.26 0.885-0.26 1.271 0 2.578 3.729 0.953 5.38-0.859 0.875-2.443 1.12-3.229 1.057-0.063 2.542-1.542 4.833-3.5 5.932-1 0.563-2.068 0.854-3.063 1.234-1.229 0.469-2.38 1.016-3.547 1.016h-0.125c-1.161-0.099-2.318-0.542-3.547-1.016-0.995-0.38-2.068-0.682-3.063-1.24-1.948-1.099-3.427-3.391-3.49-5.927-0.781 0.068-2.385-0.177-3.245-1.057-1.625-1.651-0.318-5.38 0.948-5.38 0 0 0.328-0.083 0.885 0.26 0.698 0.427 1.318 2.646 2.161 2.953 1.391-2.547 4.667-3.969 9.417-3.969zM10.875 11.422c-2.276-0.042-4.146 1.792-4.146 4.068 0 2.281 1.87 4.115 4.146 4.073 5.328-0.099 5.328-8.047 0-8.141zM21.208 11.422c-5.427 0-5.427 8.141 0 8.141s5.427-8.141 0-8.141zM11.453 13.708c2.349 0.063 2.349 3.552 0 3.615-1.182 0-2.042-1.115-1.75-2.255 0.318 0.771 1.469 0.547 1.464-0.292 0-0.406-0.318-0.745-0.729-0.76 0.302-0.203 0.656-0.313 1.016-0.307zM20.641 13.708c2.344 0.063 2.344 3.552 0 3.615-1.182 0-2.047-1.115-1.755-2.255 0.229 0.552 0.979 0.641 1.328 0.146 0.344-0.49 0.010-1.167-0.589-1.193 0.297-0.208 0.651-0.313 1.016-0.313zM15.359 19.906c-0.318 0.026-0.5 0.193-0.5 0.635 0 0.281 0.182 0.484 0.5 0.484 0.229 0 0.266-0.323 0.047-0.375-0.031-0.005-0.172-0.057-0.172-0.182 0-0.12 0-0.167 0.24-0.198 0.104-0.016 0.156-0.141 0.125-0.24s-0.125-0.135-0.24-0.125zM16.724 19.906c-0.115-0.005-0.208 0.026-0.24 0.125s0.021 0.224 0.125 0.24c0.24 0.031 0.24 0.078 0.24 0.198 0 0.125-0.141 0.177-0.172 0.182-0.219 0.052-0.182 0.375 0.042 0.375 0.323 0 0.51-0.203 0.51-0.484 0-0.443-0.188-0.609-0.505-0.635z" fill="#C5C5C5"/>
+  <line y2="24" x2="16" y1="26" x1="32" stroke-width="2" stroke="green" fill="none"/>
+  <line y2="16" x2="24" y1="32" x1="24" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/light/lgtm-plus.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none">
+  <path d="M16.010 6.49c-3.885 0-7.167 0.906-9.328 2.813-0.063-0.12-0.109-0.219-0.188-0.339-0.224-0.365-0.438-0.776-1.104-1.188-0.411-0.26-0.87-0.438-1.349-0.516-0.208-0.021-0.422-0.021-0.63 0l0.135-0.016c-1.214 0-1.922 0.724-2.385 1.354-0.458 0.625-0.755 1.328-0.948 2.099-0.38 1.542-0.385 3.536 1.083 5.026 0.766 0.781 1.667 1.151 2.484 1.37 0.156 0.042 0.297 0.052 0.448 0.083 0.531 2.521 2.104 4.656 4.208 5.839v0.005c1.24 0.693 2.417 1.010 3.297 1.349 1.234 0.479 2.536 1 4.052 1.135l0.078 0.005h0.198c1.745 0 3.063-0.703 4.203-1.141 0.875-0.333 2.052-0.641 3.302-1.344 0.578-0.323 1.115-0.719 1.594-1.172 1.318-1.234 2.229-2.839 2.625-4.599 1.115-0.182 2.141-0.719 2.922-1.536 1.464-1.484 1.458-3.479 1.078-5.021-0.193-0.771-0.49-1.474-0.948-2.099-0.458-0.63-1.172-1.354-2.385-1.354l0.135 0.016c-0.208-0.021-0.422-0.021-0.63 0-0.479 0.078-0.938 0.255-1.344 0.516-0.667 0.411-0.88 0.823-1.104 1.182-0.073 0.12-0.12 0.219-0.188 0.333-2.156-1.901-5.432-2.802-9.313-2.802zM16.042 8.313c4.745 0 8.016 1.422 9.411 3.964 0.839-0.323 1.453-2.521 2.146-2.948 0.563-0.344 0.885-0.26 0.885-0.26 1.271 0 2.578 3.729 0.953 5.38-0.859 0.875-2.443 1.12-3.229 1.057-0.063 2.542-1.542 4.833-3.5 5.932-1 0.563-2.068 0.854-3.063 1.234-1.229 0.469-2.38 1.016-3.547 1.016h-0.125c-1.161-0.099-2.318-0.542-3.547-1.016-0.995-0.38-2.068-0.682-3.063-1.24-1.948-1.099-3.427-3.391-3.49-5.927-0.781 0.068-2.385-0.177-3.245-1.057-1.625-1.651-0.318-5.38 0.948-5.38 0 0 0.328-0.083 0.885 0.26 0.698 0.427 1.318 2.646 2.161 2.953 1.391-2.547 4.667-3.969 9.417-3.969zM10.875 11.422c-2.276-0.042-4.146 1.792-4.146 4.068 0 2.281 1.87 4.115 4.146 4.073 5.328-0.099 5.328-8.047 0-8.141zM21.208 11.422c-5.427 0-5.427 8.141 0 8.141s5.427-8.141 0-8.141zM11.453 13.708c2.349 0.063 2.349 3.552 0 3.615-1.182 0-2.042-1.115-1.75-2.255 0.318 0.771 1.469 0.547 1.464-0.292 0-0.406-0.318-0.745-0.729-0.76 0.302-0.203 0.656-0.313 1.016-0.307zM20.641 13.708c2.344 0.063 2.344 3.552 0 3.615-1.182 0-2.047-1.115-1.755-2.255 0.229 0.552 0.979 0.641 1.328 0.146 0.344-0.49 0.010-1.167-0.589-1.193 0.297-0.208 0.651-0.313 1.016-0.313zM15.359 19.906c-0.318 0.026-0.5 0.193-0.5 0.635 0 0.281 0.182 0.484 0.5 0.484 0.229 0 0.266-0.323 0.047-0.375-0.031-0.005-0.172-0.057-0.172-0.182 0-0.12 0-0.167 0.24-0.198 0.104-0.016 0.156-0.141 0.125-0.24s-0.125-0.135-0.24-0.125zM16.724 19.906c-0.115-0.005-0.208 0.026-0.24 0.125s0.021 0.224 0.125 0.24c0.24 0.031 0.24 0.078 0.24 0.198 0 0.125-0.141 0.177-0.172 0.182-0.219 0.052-0.182 0.375 0.042 0.375 0.323 0 0.51-0.203 0.51-0.484 0-0.443-0.188-0.609-0.505-0.635z" fill="#424242"/>
+  <line y2="24" x2="16" y1="26" x1="32" stroke-width="2" stroke="green" fill="none"/>
+  <line y2="16" x2="24" y1="32" x1="24" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.1.5",
+  "version": "1.2.2",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -27,11 +27,15 @@
     "onView:codeQLQueryHistory",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
+    "onCommand:codeQLDatabases.chooseDatabaseFolder",
+    "onCommand:codeQLDatabases.chooseDatabaseArchive",
+    "onCommand:codeQLDatabases.chooseDatabaseInternet",
+    "onCommand:codeQLDatabases.chooseDatabaseLgtm",
+    "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
-    "onCommand:codeQL.setCurrentDatabase",
-    "onCommand:codeQL.downloadDatabase",
+    "onCommand:codeQL.chooseDatabaseLgtm",
     "onCommand:codeQLDatabases.chooseDatabase",
     "onCommand:codeQLDatabases.setCurrentDatabase",
     "onCommand:codeQL.quickQuery",
@@ -175,7 +179,7 @@
         "title": "CodeQL: Quick Query"
       },
       {
-        "command": "codeQL.chooseDatabaseFolder",
+        "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
         "icon": {
           "light": "media/light/folder-opened-plus.svg",
@@ -183,7 +187,7 @@
         }
       },
       {
-        "command": "codeQL.chooseDatabaseArchive",
+        "command": "codeQLDatabases.chooseDatabaseArchive",
         "title": "Choose Database from Archive",
         "icon": {
           "light": "media/light/archive-plus.svg",
@@ -191,13 +195,21 @@
         }
       },
       {
-        "command": "codeQL.chooseDatabaseInternet",
-        "title": "Download database",
+        "command": "codeQLDatabases.chooseDatabaseInternet",
+        "title": "Download Database",
         "icon": {
           "light": "media/light/cloud-download.svg",
           "dark": "media/dark/cloud-download.svg"
         }
       },
+      {
+        "command": "codeQLDatabases.chooseDatabaseLgtm",
+        "title": "Download from LGTM",
+        "icon": {
+          "light": "media/light/lgtm-plus.svg",
+          "dark": "media/dark/lgtm-plus.svg"
+        }
+      },
       {
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
@@ -231,8 +243,20 @@
         "title": "Show Database Directory"
       },
       {
-        "command": "codeQL.downloadDatabase",
-        "title": "CodeQL: Download database"
+        "command": "codeQL.chooseDatabaseFolder",
+        "title": "CodeQL: Choose Database from Folder"
+      },
+      {
+        "command": "codeQL.chooseDatabaseArchive",
+        "title": "CodeQL: Choose Database from Archive"
+      },
+      {
+        "command": "codeQL.chooseDatabaseInternet",
+        "title": "CodeQL: Download Database"
+      },
+      {
+        "command": "codeQL.chooseDatabaseLgtm",
+        "title": "CodeQL: Download Database from LGTM"
       },
       {
         "command": "codeQLDatabases.sortByName",
@@ -274,6 +298,10 @@
         "command": "codeQLQueryHistory.showQueryText",
         "title": "Show Query Text"
       },
+      {
+        "command": "codeQLQueryHistory.viewSarif",
+        "title": "View SARIF"
+      },
       {
         "command": "codeQLQueryResults.nextPathStep",
         "title": "CodeQL: Show Next Step on Path"
@@ -312,17 +340,22 @@
           "group": "navigation"
         },
         {
-          "command": "codeQL.chooseDatabaseFolder",
+          "command": "codeQLDatabases.chooseDatabaseFolder",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         },
         {
-          "command": "codeQL.chooseDatabaseArchive",
+          "command": "codeQLDatabases.chooseDatabaseArchive",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         },
         {
-          "command": "codeQL.chooseDatabaseInternet",
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         }
@@ -378,6 +411,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
         },
+        {
+          "command": "codeQLQueryHistory.viewSarif",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -406,10 +444,6 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
-        {
-          "command": "codeQL.downloadDatabase",
-          "when": "true"
-        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -442,6 +476,26 @@
           "command": "codeQLDatabases.removeDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.chooseDatabaseFolder",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseArchive",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.upgradeDatabase",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "when": "false"
@@ -462,6 +516,10 @@
           "command": "codeQLQueryHistory.showQueryText",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.viewSarif",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.setLabel",
           "when": "false"
@@ -532,7 +590,9 @@
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
     "vscode-test-adapter-util": "~0.7.0",
-    "minimist": "~1.2.5"
+    "minimist": "~1.2.5",
+    "semver": "~7.3.2",
+    "@types/semver": "~7.2.0"
   },
   "devDependencies": {
     "@types/chai": "^4.1.7",
@@ -575,7 +635,7 @@
     "typescript-config": "^0.0.1",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
-    "vscode-test": "^1.0.0",
+    "vscode-test": "^1.4.0",
     "webpack": "^4.38.0",
     "webpack-cli": "^3.3.2",
     "eslint": "~6.8.0",

extensions/ql-vscode/src/adapt.ts

@@ -101,3 +101,34 @@ export function adaptBqrs(schema: AdaptedSchema, page: DecodedBqrsChunk): RawRes
     rows: page.tuples.map(adaptRow),
   };
 }
+
+/**
+ * This type has two branches; we are in the process of changing from
+ * one to the other. The old way is to parse them inside the webview,
+ * the new way is to parse them in the extension. The main motivation
+ * for this transition is to make pagination possible in such a way
+ * that only one page needs to be sent from the extension to the webview.
+ */
+export type ParsedResultSets = ExtensionParsedResultSets | WebviewParsedResultSets;
+
+/**
+ * The old method doesn't require any nontrivial information to be included here,
+ * just a tag to indicate that it is being used.
+ */
+export interface WebviewParsedResultSets {
+  t: 'WebviewParsed';
+  selectedTable?: string; // when undefined, means 'show default table'
+}
+
+/**
+ * The new method includes which bqrs page is being sent, and the
+ * actual results parsed on the extension side.
+ */
+export interface ExtensionParsedResultSets {
+  t: 'ExtensionParsed';
+  pageNumber: number;
+  numPages: number;
+  selectedTable?: string; // when undefined, means 'show default table'
+  resultSetNames: string[];
+  resultSet: RawResultSet;
+}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -173,7 +173,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
     const ref = decodeSourceArchiveUri(uri);
     const archive = await this.getArchive(ref.sourceArchiveZipPath);
     const contents = archive.dirMap.get(ref.pathWithinSourceArchive);
-    const result = contents === undefined ? [] : Array.from(contents.entries());
+    const result = contents === undefined ? undefined : Array.from(contents.entries());
     if (result === undefined) {
       throw vscode.FileSystemError.FileNotFound(uri);
     }
@@ -238,7 +238,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
     if (archive.dirMap.has(reqPath)) {
       return new Directory(reqPath);
     }
-    throw vscode.FileSystemError.FileNotFound(uri);
+    throw vscode.FileSystemError.FileNotFound(`uri '${uri.toString()}', interpreted as '${reqPath}' in archive '${ref.sourceArchiveZipPath}'`);
   }
 
   private async _lookupAsFile(uri: vscode.Uri): Promise<File> {

extensions/ql-vscode/src/cli-version.ts

@@ -1,10 +1,11 @@
+import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
 import { Logger } from "./logging";
 
 /**
  * Get the version of a CodeQL CLI.
  */
-export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<Version | undefined> {
+export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<semver.SemVer | undefined> {
   const output: string = await runCodeQlCliCommand(
     codeQlPath,
     ["version"],
@@ -12,85 +13,5 @@ export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): P
     "Checking CodeQL version",
     logger
   );
-  return tryParseVersionString(output.trim());
-}
-
-/**
- * Try to parse a version string, returning undefined if we can't parse it.
- * 
- * Version strings must contain a major, minor, and patch version.  They may optionally
- * start with "v" and may optionally contain some "tail" string after the major, minor, and
- * patch versions, for example as in `v2.1.0+baf5bff`.
- */
-export function tryParseVersionString(versionString: string): Version | undefined {
-  const match = versionString.match(versionRegex);
-  if (match === null) {
-    return undefined;
-  }
-  return {
-    buildMetadata: match[5],
-    majorVersion: Number.parseInt(match[1], 10),
-    minorVersion: Number.parseInt(match[2], 10),
-    patchVersion: Number.parseInt(match[3], 10),
-    prereleaseVersion: match[4],
-    rawString: versionString,
-  };
-}
-
-/**
- * Regex for parsing semantic versions
- * 
- * From the semver spec https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
- */
-const versionRegex = new RegExp(String.raw`^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)` +
-  String.raw`(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?` +
-  String.raw`(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`);
-
-/**
- * A version of the CodeQL CLI.
- */
-export interface Version {
-  /**
-   * Build metadata
-   * 
-   * For example, this will be `abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   * Build metadata must be ignored when comparing versions. 
-   */
-  buildMetadata: string | undefined;
-
-  /**
-   * Major version number
-   * 
-   * For example, this will be `2` for version 2.1.0-alpha.1+abcdef0.
-   */
-  majorVersion: number;
-
-  /**
-   * Minor version number
-   * 
-   * For example, this will be `1` for version 2.1.0-alpha.1+abcdef0.
-   */
-  minorVersion: number;
-
-  /**
-   * Patch version number
-   * 
-   * For example, this will be `0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  patchVersion: number;
-
-  /**
-   * Prerelease version
-   * 
-   * For example, this will be `alpha.1` for version 2.1.0-alpha.1+abcdef0.
-   * The prerelease version must be considered when comparing versions.
-   */
-  prereleaseVersion: string | undefined;
-
-  /**
-   * Raw version string
-   * 
-   * For example, this will be `2.1.0-alpha.1+abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  rawString: string;
+  return semver.parse(output.trim()) || undefined;
 }

extensions/ql-vscode/src/config.ts

@@ -42,13 +42,11 @@ const ROOT_SETTING = new Setting('codeQL');
 // Enable experimental features
 
 /**
- * This setting is deliberately not in package.json so that it does
- * not appear in the settings ui in vscode itself. If users want to
- * enable experimental features, they can add
- * "codeQl.experimentalFeatures" directly in their vscode settings
- * json file.
+ * Any settings below are deliberately not in package.json so that
+ * they do not appear in the settings ui in vscode itself. If users
+ * want to enable experimental features, they can add them directly in
+ * their vscode settings json file.
  */
-export const EXPERIMENTAL_FEATURES_SETTING = new Setting('experimentalFeatures', ROOT_SETTING);
 
 /* Advanced setting: used to enable bqrs parsing in the cli instead of in the webview. */
 export const EXPERIMENTAL_BQRS_SETTING = new Setting('experimentalBqrsParsing', ROOT_SETTING);

extensions/ql-vscode/src/databaseFetcher.ts

@@ -1,10 +1,22 @@
-import * as fetch from "node-fetch";
+import fetch, { Response } from "node-fetch";
 import * as unzipper from "unzipper";
-import { Uri, ProgressOptions, ProgressLocation, commands, window } from "vscode";
+import {
+  Uri,
+  ProgressOptions,
+  ProgressLocation,
+  commands,
+  window,
+} from "vscode";
 import * as fs from "fs-extra";
 import * as path from "path";
 import { DatabaseManager, DatabaseItem } from "./databases";
-import { ProgressCallback, showAndLogErrorMessage, withProgress, showAndLogInformationMessage } from "./helpers";
+import {
+  ProgressCallback,
+  showAndLogErrorMessage,
+  withProgress,
+  showAndLogInformationMessage,
+} from "./helpers";
+import { logger } from "./logging";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -12,25 +24,39 @@ import { ProgressCallback, showAndLogErrorMessage, withProgress, showAndLogInfor
  * @param databasesManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
-export async function promptImportInternetDatabase(databasesManager: DatabaseManager, storagePath: string): Promise<DatabaseItem | undefined>  {
+export async function promptImportInternetDatabase(
+  databasesManager: DatabaseManager,
+  storagePath: string
+): Promise<DatabaseItem | undefined> {
   let item: DatabaseItem | undefined = undefined;
 
   try {
     const databaseUrl = await window.showInputBox({
-      prompt: 'Enter URL of zipfile of database to download'
+      prompt: "Enter URL of zipfile of database to download",
     });
     if (databaseUrl) {
       validateHttpsUrl(databaseUrl);
 
       const progressOptions: ProgressOptions = {
         location: ProgressLocation.Notification,
-        title: 'Adding database from URL',
+        title: "Adding database from URL",
         cancellable: false,
       };
-      await withProgress(progressOptions, async progress => (item = await databaseArchiveFetcher(databaseUrl, databasesManager, storagePath, progress)));
-      commands.executeCommand('codeQLDatabases.focus');
+      await withProgress(
+        progressOptions,
+        async (progress) =>
+          (item = await databaseArchiveFetcher(
+            databaseUrl,
+            databasesManager,
+            storagePath,
+            progress
+          ))
+      );
+      commands.executeCommand("codeQLDatabases.focus");
     }
-    showAndLogInformationMessage('Database downloaded and imported successfully.');
+    showAndLogInformationMessage(
+      "Database downloaded and imported successfully."
+    );
   } catch (e) {
     showAndLogErrorMessage(e.message);
   }
@@ -38,6 +64,62 @@ export async function promptImportInternetDatabase(databasesManager: DatabaseMan
   return item;
 }
 
+/**
+ * Prompts a user to fetch a database from lgtm.
+ * User enters a project url and then the user is asked which language
+ * to download (if there is more than one)
+ *
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function promptImportLgtmDatabase(
+  databasesManager: DatabaseManager,
+  storagePath: string
+): Promise<DatabaseItem | undefined> {
+  let item: DatabaseItem | undefined = undefined;
+
+  try {
+    const lgtmUrl = await window.showInputBox({
+      prompt:
+        "Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)",
+    });
+    if (!lgtmUrl) {
+      return;
+    }
+    if (looksLikeLgtmUrl(lgtmUrl)) {
+      const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
+      if (databaseUrl) {
+        const progressOptions: ProgressOptions = {
+          location: ProgressLocation.Notification,
+          title: "Adding database from LGTM",
+          cancellable: false,
+        };
+        await withProgress(
+          progressOptions,
+          async (progress) =>
+            (item = await databaseArchiveFetcher(
+              databaseUrl,
+              databasesManager,
+              storagePath,
+              progress
+            ))
+        );
+        commands.executeCommand("codeQLDatabases.focus");
+      }
+    } else {
+      throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+    }
+    if (item) {
+      showAndLogInformationMessage(
+        "Database downloaded and imported successfully."
+      );
+    }
+  } catch (e) {
+    showAndLogErrorMessage(e.message);
+  }
+
+  return item;
+}
 
 /**
  * Imports a database from a local archive.
@@ -46,25 +128,41 @@ export async function promptImportInternetDatabase(databasesManager: DatabaseMan
  * @param databasesManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
-export async function importArchiveDatabase(databaseUrl: string, databasesManager: DatabaseManager, storagePath: string): Promise<DatabaseItem | undefined>  {
+export async function importArchiveDatabase(
+  databaseUrl: string,
+  databasesManager: DatabaseManager,
+  storagePath: string
+): Promise<DatabaseItem | undefined> {
   let item: DatabaseItem | undefined = undefined;
   try {
     const progressOptions: ProgressOptions = {
       location: ProgressLocation.Notification,
-      title: 'Importing database from archive',
+      title: "Importing database from archive",
       cancellable: false,
     };
-    await withProgress(progressOptions, async progress => (item = await databaseArchiveFetcher(databaseUrl, databasesManager, storagePath, progress)));
-    commands.executeCommand('codeQLDatabases.focus');
+    await withProgress(
+      progressOptions,
+      async (progress) =>
+        (item = await databaseArchiveFetcher(
+          databaseUrl,
+          databasesManager,
+          storagePath,
+          progress
+        ))
+    );
+    commands.executeCommand("codeQLDatabases.focus");
 
-    showAndLogInformationMessage('Database unzipped and imported successfully.');
+    if (item) {
+      showAndLogInformationMessage(
+        "Database unzipped and imported successfully."
+      );
+    }
   } catch (e) {
     showAndLogErrorMessage(e.message);
   }
   return item;
 }
 
-
 /**
  * Fetches an archive database. The database might be on the internet
  * or in the local filesystem.
@@ -82,8 +180,8 @@ async function databaseArchiveFetcher(
 ): Promise<DatabaseItem> {
   progressCallback?.({
     maxStep: 3,
-    message: 'Getting database',
-    step: 1
+    message: "Getting database",
+    step: 1,
   });
   if (!storagePath) {
     throw new Error("No storage path specified.");
@@ -99,30 +197,41 @@ async function databaseArchiveFetcher(
 
   progressCallback?.({
     maxStep: 3,
-    message: 'Opening database',
-    step: 3
+    message: "Opening database",
+    step: 3,
   });
 
   // find the path to the database. The actual database might be in a sub-folder
-  const dbPath = await findDirWithFile(unzipPath, '.dbinfo', 'codeql-database.yml');
+  const dbPath = await findDirWithFile(
+    unzipPath,
+    ".dbinfo",
+    "codeql-database.yml"
+  );
   if (dbPath) {
-    const item = await databasesManager.openDatabase(Uri.parse(dbPath));
+    const item = await databasesManager.openDatabase(Uri.file(dbPath));
     databasesManager.setCurrentDatabaseItem(item);
     return item;
   } else {
-    throw new Error('Database not found in archive.');
+    throw new Error("Database not found in archive.");
   }
 }
 
 async function getStorageFolder(storagePath: string, urlStr: string) {
+  // we need to generate a folder name for the unzipped archive,
+  // this needs to be human readable since we may use this name as the initial
+  // name for the database
   const url = Uri.parse(urlStr);
-  let lastName = path.basename(url.path).substring(0, 255);
+  // MacOS has a max filename length of 255
+  // and remove a few extra chars in case we need to add a counter at the end.
+  let lastName = path.basename(url.path).substring(0, 250);
   if (lastName.endsWith(".zip")) {
     lastName = lastName.substring(0, lastName.length - 4);
   }
 
   const realpath = await fs.realpath(storagePath);
   let folderName = path.join(realpath, lastName);
+
+  // avoid overwriting existing folders
   let counter = 0;
   while (await fs.pathExists(folderName)) {
     counter++;
@@ -134,7 +243,6 @@ async function getStorageFolder(storagePath: string, urlStr: string) {
   return folderName;
 }
 
-
 function validateHttpsUrl(databaseUrl: string) {
   let uri;
   try {
@@ -143,47 +251,78 @@ function validateHttpsUrl(databaseUrl: string) {
     throw new Error(`Invalid url: ${databaseUrl}`);
   }
 
-  if (uri.scheme !== 'https') {
-    throw new Error('Must use https for downloading a database.');
+  if (uri.scheme !== "https") {
+    throw new Error("Must use https for downloading a database.");
   }
 }
 
 async function readAndUnzip(databaseUrl: string, unzipPath: string) {
   const unzipStream = unzipper.Extract({
-    path: unzipPath
+    path: unzipPath,
   });
 
   await new Promise((resolve, reject) => {
     // we already know this is a file scheme
-    const databaseFile = Uri.parse(databaseUrl).path;
+    const databaseFile = Uri.parse(databaseUrl).fsPath;
     const stream = fs.createReadStream(databaseFile);
-    stream.on('error', reject);
-    unzipStream.on('error', reject);
-    unzipStream.on('close', resolve);
+    stream.on("error", reject);
+    unzipStream.on("error", reject);
+    unzipStream.on("close", resolve);
     stream.pipe(unzipStream);
   });
 }
 
-async function fetchAndUnzip(databaseUrl: string, unzipPath: string, progressCallback?: ProgressCallback) {
-  const response = await fetch.default(databaseUrl);
+async function fetchAndUnzip(
+  databaseUrl: string,
+  unzipPath: string,
+  progressCallback?: ProgressCallback
+) {
+  const response = await fetch(databaseUrl);
+
+  await checkForFailingResponse(response);
+
   const unzipStream = unzipper.Extract({
-    path: unzipPath
+    path: unzipPath,
   });
   progressCallback?.({
     maxStep: 3,
-    message: 'Unzipping database',
-    step: 2
+    message: "Unzipping database",
+    step: 2,
   });
   await new Promise((resolve, reject) => {
-    response.body.on('error', reject);
-    unzipStream.on('error', reject);
-    unzipStream.on('close', resolve);
+    const handler = (err: Error) => {
+      if (err.message.startsWith('invalid signature')) {
+        reject(new Error('Not a valid archive.'));
+      } else {
+        reject(err);
+      }
+    };
+    response.body.on("error", handler);
+    unzipStream.on("error", handler);
+    unzipStream.on("close", resolve);
     response.body.pipe(unzipStream);
   });
 }
 
+async function checkForFailingResponse(response: Response): Promise<void | never> {
+  if (response.ok) {
+    return;
+  }
+
+  // An error downloading the database. Attempt to extract the resaon behind it.
+  const text = await response.text();
+  let msg: string;
+  try {
+    const obj = JSON.parse(text);
+    msg = obj.error || obj.message || obj.reason || JSON.stringify(obj, null, 2);
+  } catch (e) {
+    msg = text;
+  }
+  throw new Error(`Error downloading database.\n\nReason: ${msg}`);
+}
+
 function isFile(databaseUrl: string) {
-  return Uri.parse(databaseUrl).scheme === 'file';
+  return Uri.parse(databaseUrl).scheme === "file";
 }
 
 /**
@@ -194,12 +333,16 @@ function isFile(databaseUrl: string) {
  *
  * @returns the directory containing the file, or undefined if not found.
  */
-async function findDirWithFile(dir: string, ...toFind: string[]): Promise<string | undefined> {
+// exported for testing
+export async function findDirWithFile(
+  dir: string,
+  ...toFind: string[]
+): Promise<string | undefined> {
   if (!(await fs.stat(dir)).isDirectory()) {
     return;
   }
   const files = await fs.readdir(dir);
-  if (toFind.some(file => files.includes(file))) {
+  if (toFind.some((file) => files.includes(file))) {
     return dir;
   }
   for (const file of files) {
@@ -211,3 +354,89 @@ async function findDirWithFile(dir: string, ...toFind: string[]): Promise<string
   }
   return;
 }
+
+/**
+ * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
+ * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
+ *
+ * After the {provider}/{org}/{name} path components, there may be the components
+ * related to sub pages.
+ *
+ * This function accepts any url that matches the patter above
+ *
+ * @param lgtmUrl The URL to the lgtm project
+ *
+ * @return true if this looks like an LGTM project url
+ */
+// exported for testing
+export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string {
+  if (!lgtmUrl) {
+    return false;
+  }
+
+  try {
+    const uri = Uri.parse(lgtmUrl, true);
+    if (uri.scheme !== "https") {
+      return false;
+    }
+
+    if (uri.authority !== "lgtm.com" && uri.authority !== "www.lgtm.com") {
+      return false;
+    }
+
+    const paths = uri.path.split("/").filter((segment) => segment);
+    return paths.length >= 4 && paths[0] === "projects";
+  } catch (e) {
+    return false;
+  }
+}
+
+// exported for testing
+export async function convertToDatabaseUrl(lgtmUrl: string) {
+  try {
+    const uri = Uri.parse(lgtmUrl, true);
+    const paths = ["api", "v1.0"].concat(
+      uri.path.split("/").filter((segment) => segment)
+    ).slice(0, 6);
+    const projectUrl = `https://lgtm.com/${paths.join("/")}`;
+    const projectResponse = await fetch(projectUrl);
+    const projectJson = await projectResponse.json();
+
+    if (projectJson.code === 404) {
+      throw new Error();
+    }
+
+    const language = await promptForLanguage(projectJson);
+    if (!language) {
+      return;
+    }
+    return `https://lgtm.com/${[
+      "api",
+      "v1.0",
+      "snapshots",
+      projectJson.id,
+      language,
+    ].join("/")}`;
+  } catch (e) {
+    logger.log(`Error: ${e.message}`);
+    throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+  }
+}
+
+async function promptForLanguage(
+  projectJson: any
+): Promise<string | undefined> {
+  if (!projectJson?.languages?.length) {
+    return;
+  }
+  if (projectJson.languages.length === 1) {
+    return projectJson.languages[0].language;
+  }
+
+  return await window.showQuickPick(
+    projectJson.languages.map((lang: { language: string }) => lang.language), {
+    placeHolder: "Select the database language to download:"
+  }
+  );
+}

extensions/ql-vscode/src/databases-ui.ts

@@ -8,7 +8,8 @@ import { logger } from './logging';
 import { clearCacheInDatabase, UserCancellationException } from './run-queries';
 import * as qsClient from './queryserver-client';
 import { upgradeDatabase } from './upgrades';
-import { importArchiveDatabase, promptImportInternetDatabase } from './databaseFetcher';
+import { importArchiveDatabase, promptImportInternetDatabase, promptImportLgtmDatabase } from './databaseFetcher';
+import * as fs from 'fs-extra';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -174,9 +175,11 @@ export class DatabaseUI extends DisposableObject {
     this.treeDataProvider = this.push(new DatabaseTreeDataProvider(ctx, databaseManager));
     this.push(window.createTreeView('codeQLDatabases', { treeDataProvider: this.treeDataProvider }));
 
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseFolder', this.handleChooseDatabaseFolder));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseArchive', this.handleChooseDatabaseArchive));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseInternet', this.handleChooseDatabaseInternet));
+    logger.log('Registering database panel commands.');
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseFolder', this.handleChooseDatabaseFolder));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseArchive', this.handleChooseDatabaseArchive));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseInternet', this.handleChooseDatabaseInternet));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseLgtm', this.handleChooseDatabaseLgtm));
     ctx.subscriptions.push(commands.registerCommand('codeQL.setCurrentDatabase', this.handleSetCurrentDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQL.upgradeCurrentDatabase', this.handleUpgradeCurrentDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQL.clearCache', this.handleClearCache));
@@ -193,7 +196,7 @@ export class DatabaseUI extends DisposableObject {
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
   }
 
-  private handleChooseDatabaseFolder = async (): Promise<DatabaseItem | undefined> => {
+  handleChooseDatabaseFolder = async (): Promise<DatabaseItem | undefined> => {
     try {
       return await this.chooseAndSetDatabase(true);
     } catch (e) {
@@ -202,7 +205,7 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
-  private handleChooseDatabaseArchive = async (): Promise<DatabaseItem | undefined> => {
+  handleChooseDatabaseArchive = async (): Promise<DatabaseItem | undefined> => {
     try {
       return await this.chooseAndSetDatabase(false);
     } catch (e) {
@@ -211,10 +214,14 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
-  private handleChooseDatabaseInternet = async (): Promise<DatabaseItem | undefined> => {
+  handleChooseDatabaseInternet = async (): Promise<DatabaseItem | undefined> => {
     return await promptImportInternetDatabase(this.databaseManager, this.storagePath);
   }
 
+  handleChooseDatabaseLgtm = async (): Promise<DatabaseItem | undefined> => {
+    return await promptImportLgtmDatabase(this.databaseManager, this.storagePath);
+  }
+
   private handleSortByName = async () => {
     if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
       this.treeDataProvider.sortOrder = SortOrder.NameDesc;
@@ -292,7 +299,7 @@ export class DatabaseUI extends DisposableObject {
   private handleSetCurrentDatabase = async (uri: Uri): Promise<DatabaseItem | undefined> => {
     // Assume user has selected an archive if the file has a .zip extension
     if (uri.path.endsWith('.zip')) {
-      return await importArchiveDatabase(uri.toString(), this.databaseManager, this.storagePath);
+      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
     }
 
     return await this.setCurrentDatabase(uri);
@@ -360,13 +367,36 @@ export class DatabaseUI extends DisposableObject {
     }
 
     if (byFolder) {
+      const fixedUri = await this.fixDbUri(uri);
       // we are selecting a database folder
-      return await this.setCurrentDatabase(uri);
+      return await this.setCurrentDatabase(fixedUri);
     }
     else {
       // we are selecting a database archive. Must unzip into a workspace-controlled area
       // before importing.
-      return await importArchiveDatabase(uri.toString(), this.databaseManager, this.storagePath);
+      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
     }
   }
+
+  /**
+   * Perform some heuristics to ensure a proper database location is chosen.
+   *
+   * 1. If the selected URI to add is a file, choose the containing directory
+   * 2. If the selected URI is a directory matching db-*, choose the containing directory
+   * 3. choose the current directory
+   *
+   * @param uri a URI that is a datbase folder or inside it
+   *
+   * @return the actual database folder found by using the heuristics above.
+   */
+  private async fixDbUri(uri: Uri): Promise<Uri> {
+    let dbPath = uri.fsPath;
+    if ((await fs.stat(dbPath)).isFile()) {
+      dbPath = path.dirname(dbPath);
+    }
+    if (path.basename(dbPath).startsWith('db-')) {
+      dbPath = path.dirname(dbPath);
+    }
+    return Uri.file(dbPath);
+  }
 }

extensions/ql-vscode/src/definitions.ts

@@ -34,6 +34,13 @@ function tagOfKeyType(keyType: KeyType): string {
   }
 }
 
+function nameOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery: return "definitions";
+    case KeyType.ReferenceQuery: return "references";
+  }
+}
+
 async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
   const suiteFile = tmp.fileSync({ postfix: '.qls' }).name;
   const suiteYaml = { qlpack, include: { kind: 'definitions', 'tags contain': tagOfKeyType(keyType) } };
@@ -41,7 +48,10 @@ async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: Key
 
   const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
   if (queries.length === 0) {
-    throw new Error("Couldn't find any queries for qlpack");
+    vscode.window.showErrorMessage(
+      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. It might be necessary to upgrade the CodeQL libraries.`
+    );
+    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
   }
   return queries;
 }

extensions/ql-vscode/src/distribution.ts

@@ -2,6 +2,7 @@ import * as fetch from "node-fetch";
 import * as fs from "fs-extra";
 import * as os from "os";
 import * as path from "path";
+import * as semver from "semver";
 import * as unzipper from "unzipper";
 import * as url from "url";
 import { ExtensionContext, Event } from "vscode";
@@ -9,7 +10,7 @@ import { DistributionConfig } from "./config";
 import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from "./helpers";
 import { logger } from "./logging";
 import * as helpers from "./helpers";
-import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-version";
+import { getCodeQlCliVersion } from "./cli-version";
 
 /**
  * distribution.ts
@@ -35,16 +36,11 @@ const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";
 const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-binaries";
 
 /**
- * Version constraint for the CLI.
+ * Range of versions of the CLI that are compatible with the extension.
  *
  * This applies to both extension-managed and CLI distributions.
  */
-export const DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT: VersionConstraint = {
-  description: "2.*.*",
-  isVersionCompatible: (v: Version) => {
-    return v.majorVersion === 2 && v.minorVersion >= 0;
-  }
-};
+export const DEFAULT_DISTRIBUTION_VERSION_RANGE: semver.Range = new semver.Range("2.x");
 
 export interface DistributionProvider {
   getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
@@ -52,44 +48,63 @@ export interface DistributionProvider {
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
+  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
     this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionConstraint);
+    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
     this._onDidChangeDistribution = config.onDidChangeDistributionConfiguration;
     this._updateCheckRateLimiter = new InvocationRateLimiter(
       extensionContext,
       "extensionSpecificDistributionUpdateCheck",
       () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
     );
-    this._versionConstraint = versionConstraint;
+    this._versionRange = versionRange;
   }
 
   /**
    * Look up a CodeQL launcher binary.
    */
   public async getDistribution(): Promise<FindDistributionResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath === undefined) {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    if (distribution === undefined) {
       return {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(codeQlPath, logger);
-    if (version !== undefined && !this._versionConstraint.isVersionCompatible(version)) {
+    const version = await getCodeQlCliVersion(distribution.codeQlPath, logger);
+    if (version === undefined) {
       return {
-        codeQlPath,
+        distribution,
+        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
+      };
+    }
+
+    /**
+     * Specifies whether prerelease versions of the CodeQL CLI should be accepted.
+     * 
+     * Suppose a user sets the includePrerelease config option, obtains a prerelease, then decides
+     * they no longer want a prerelease, so unsets the includePrerelease config option.
+     * Unsetting the includePrerelease config option should trigger an update check, and this
+     * update check should present them an update that returns them back to a non-prerelease
+     * version.
+     * 
+     * Therefore, we adopt the following:
+     * 
+     * - If the user is managing their own CLI, they can use a prerelease without specifying the
+     * includePrerelease option.
+     * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
+     * includePrerelease config option is set.
+     */
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+
+    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+      return {
+        distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
         version,
       };
     }
-    if (version === undefined) {
-      return {
-        codeQlPath,
-        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
-      };
-    }
     return {
-      codeQlPath,
+      distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
       version
     };
@@ -100,10 +115,15 @@ export class DistributionManager implements DistributionProvider {
     return result.kind !== FindDistributionResultKind.NoDistribution;
   }
 
+  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    return distribution?.codeQlPath;
+  }
+
   /**
    * Returns the path to a possibly-compatible CodeQL launcher binary, or undefined if a binary not be found.
    */
-  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+  async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
     if (this._config.customCodeQlPath) {
       if (!await fs.pathExists(this._config.customCodeQlPath)) {
@@ -121,19 +141,28 @@ export class DistributionManager implements DistributionProvider {
       ) {
         warnDeprecatedLauncher();
       }
-      return this._config.customCodeQlPath;
+      return {
+        codeQlPath: this._config.customCodeQlPath,
+        kind: DistributionKind.CustomPathConfig
+      };
     }
 
     const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
-      return extensionSpecificCodeQlPath;
+      return {
+        codeQlPath: extensionSpecificCodeQlPath,
+        kind: DistributionKind.ExtensionManaged
+      };
     }
 
     if (process.env.PATH) {
       for (const searchDirectory of process.env.PATH.split(path.delimiter)) {
         const expectedLauncherPath = await getExecutableFromDirectory(searchDirectory);
         if (expectedLauncherPath) {
-          return expectedLauncherPath;
+          return {
+            codeQlPath: expectedLauncherPath,
+            kind: DistributionKind.PathEnvironmentVariable
+          };
         }
       }
       logger.log("INFO: Could not find CodeQL on path.");
@@ -150,9 +179,9 @@ export class DistributionManager implements DistributionProvider {
    */
   public async checkForUpdatesToExtensionManagedDistribution(
     minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
+    const distribution = await this.getDistributionWithoutVersionCheck();
     const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath !== undefined && codeQlPath !== extensionManagedCodeQlPath) {
+    if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
       return createInvalidLocationResult();
     }
@@ -198,14 +227,14 @@ export class DistributionManager implements DistributionProvider {
   private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
   private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionConstraint: VersionConstraint;
+  private readonly _versionRange: semver.Range;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
+  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
     this._extensionContext = extensionContext;
     this._config = config;
-    this._versionConstraint = versionConstraint;
+    this._versionRange = versionRange;
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@@ -268,7 +297,18 @@ class ExtensionSpecificDistributionManager {
         `but encountered an error: ${e}.`);
     }
 
-    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(release.assets[0]);
+    // Filter assets to the unique one that we require.
+    const requiredAssetName = this.getRequiredAssetName();
+    const assets = release.assets.filter(asset => asset.name === requiredAssetName);
+    if (assets.length === 0) {
+      throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
+    }
+    if (assets.length > 1) {
+      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+        assets.map(asset => asset.name).join(', '));
+    }
+
+    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(assets[0]);
     const tmpDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-codeql"));
 
     try {
@@ -325,12 +365,36 @@ class ExtensionSpecificDistributionManager {
     }
   }
 
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  private getRequiredAssetName(): string {
+    if (os.platform() === 'linux') return 'codeql-linux64.zip';
+    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
+    if (os.platform() === 'win32') return 'codeql-win64.zip';
+    return 'codeql.zip';
+  }
+
   private async getLatestRelease(): Promise<Release> {
-    const release = await this.createReleasesApiConsumer().getLatestRelease(this._versionConstraint, this._config.includePrerelease);
-    if (release.assets.length !== 1) {
-      throw new Error("Release had an unexpected number of assets");
-    }
-    return release;
+    const requiredAssetName = this.getRequiredAssetName();
+    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    return this.createReleasesApiConsumer().getLatestRelease(
+      this._versionRange,
+      this._config.includePrerelease,
+      release => {
+        const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
+        if (matchingAssets.length === 0) {
+          // For example, this could be a release with no platform-specific assets.
+          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          return false;
+        }
+        if (matchingAssets.length > 1) {
+          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          return false;
+        }
+        return true;
+      }
+    );
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
@@ -369,7 +433,7 @@ class ExtensionSpecificDistributionManager {
 
   private readonly _config: DistributionConfig;
   private readonly _extensionContext: ExtensionContext;
-  private readonly _versionConstraint: VersionConstraint;
+  private readonly _versionRange: semver.Range;
 
   private static readonly _currentDistributionFolderBaseName = "distribution";
   private static readonly _currentDistributionFolderIndexStateKey = "distributionFolderIndex";
@@ -390,7 +454,7 @@ export class ReleasesApiConsumer {
     this._repoName = repoName;
   }
 
-  public async getLatestRelease(versionConstraint: VersionConstraint, includePrerelease = false): Promise<Release> {
+  public async getLatestRelease(versionRange: semver.Range, includePrerelease = false, additionalCompatibilityCheck?: (release: GithubRelease) => boolean): Promise<Release> {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases`;
     const allReleases: GithubRelease[] = await (await this.makeApiCall(apiPath)).json();
     const compatibleReleases = allReleases.filter(release => {
@@ -398,20 +462,20 @@ export class ReleasesApiConsumer {
         return false;
       }
 
-      const version = tryParseVersionString(release.tag_name);
-      if (version === undefined || !versionConstraint.isVersionCompatible(version)) {
+      const version = semver.parse(release.tag_name);
+      if (version === null || !semver.satisfies(version, versionRange, { includePrerelease })) {
         return false;
       }
 
-      return true;
+      return !additionalCompatibilityCheck || additionalCompatibilityCheck(release);
     });
-    // tryParseVersionString must succeed due to the previous filtering step
+    // Tag names must all be parsable to semvers due to the previous filtering step.
     const latestRelease = compatibleReleases.sort((a, b) => {
-      const versionComparison = versionCompare(tryParseVersionString(b.tag_name)!, tryParseVersionString(a.tag_name)!);
-      if (versionComparison === 0) {
-        return b.created_at.localeCompare(a.created_at);
+      const versionComparison = semver.compare(semver.parse(b.tag_name)!, semver.parse(a.tag_name)!);
+      if (versionComparison !== 0) {
+        return versionComparison;
       }
-      return versionComparison;
+      return b.created_at.localeCompare(a.created_at, "en-US");
     })[0];
     if (latestRelease === undefined) {
       throw new Error("No compatible CodeQL CLI releases were found. " +
@@ -511,29 +575,6 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-/**
- * Comparison of semantic versions.
- *
- * Returns a positive number if a is greater than b.
- * Returns 0 if a equals b.
- * Returns a negative number if a is less than b.
- */
-export function versionCompare(a: Version, b: Version): number {
-  if (a.majorVersion !== b.majorVersion) {
-    return a.majorVersion - b.majorVersion;
-  }
-  if (a.minorVersion !== b.minorVersion) {
-    return a.minorVersion - b.minorVersion;
-  }
-  if (a.patchVersion !== b.patchVersion) {
-    return a.patchVersion - b.patchVersion;
-  }
-  if (a.prereleaseVersion !== undefined && b.prereleaseVersion !== undefined) {
-    return a.prereleaseVersion.localeCompare(b.prereleaseVersion);
-  }
-  return 0;
-}
-
 function codeQlLauncherName(): string {
   return (os.platform() === "win32") ? "codeql.exe" : "codeql";
 }
@@ -550,6 +591,17 @@ function isRedirectStatusCode(statusCode: number): boolean {
  * Types and helper functions relating to those types.
  */
 
+export enum DistributionKind {
+  CustomPathConfig,
+  ExtensionManaged,
+  PathEnvironmentVariable
+}
+
+export interface Distribution {
+  codeQlPath: string;
+  kind: DistributionKind;
+}
+
 export enum FindDistributionResultKind {
   CompatibleDistribution,
   UnknownCompatibilityDistribution,
@@ -563,21 +615,27 @@ export type FindDistributionResult =
   | IncompatibleDistributionResult
   | NoDistributionResult;
 
-interface CompatibleDistributionResult {
-  codeQlPath: string;
-  kind: FindDistributionResultKind.CompatibleDistribution;
-  version: Version;
+/**
+ * A result representing a distribution of the CodeQL CLI that may or may not be compatible with
+ * the extension.
+ */
+interface DistributionResult {
+  distribution: Distribution;
+  kind: FindDistributionResultKind;
 }
 
-interface UnknownCompatibilityDistributionResult {
-  codeQlPath: string;
+interface CompatibleDistributionResult extends DistributionResult {
+  kind: FindDistributionResultKind.CompatibleDistribution;
+  version: semver.SemVer;
+}
+
+interface UnknownCompatibilityDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.UnknownCompatibilityDistribution;
 }
 
-interface IncompatibleDistributionResult {
-  codeQlPath: string;
+interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: Version;
+  version: semver.SemVer;
 }
 
 interface NoDistributionResult {
@@ -717,7 +775,7 @@ export interface GithubRelease {
   assets: GithubReleaseAsset[];
 
   /**
-   * The creation date of the release on GitHub.
+   * The creation date of the release on GitHub, in ISO 8601 format.
    */
   created_at: string;
 
@@ -762,11 +820,6 @@ export interface GithubReleaseAsset {
   size: number;
 }
 
-interface VersionConstraint {
-  description: string;
-  isVersionCompatible(version: Version): boolean;
-}
-
 export class GithubApiError extends Error {
   constructor(public status: number, public body: string) {
     super(`API call failed with status code ${status}, body: ${body}`);

extensions/ql-vscode/src/extension.ts

@@ -3,11 +3,21 @@ import { LanguageClient } from 'vscode-languageclient';
 import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
 import { CodeQLCliServer } from './cli';
-import { DistributionConfigListener, QueryHistoryConfigListener, QueryServerConfigListener, EXPERIMENTAL_FEATURES_SETTING } from './config';
+import { DistributionConfigListener, QueryHistoryConfigListener, QueryServerConfigListener } from './config';
+import * as languageSupport from './languageSupport';
 import { DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
 import { TemplateQueryDefinitionProvider, TemplateQueryReferenceProvider } from './definitions';
-import { DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT, DistributionManager, DistributionUpdateCheckResultKind, FindDistributionResult, FindDistributionResultKind, GithubApiError, GithubRateLimitedError } from './distribution';
+import {
+  DEFAULT_DISTRIBUTION_VERSION_RANGE,
+  DistributionKind,
+  DistributionManager,
+  DistributionUpdateCheckResultKind,
+  FindDistributionResult,
+  FindDistributionResultKind,
+  GithubApiError,
+  GithubRateLimitedError
+} from './distribution';
 import * as helpers from './helpers';
 import { assertNever } from './helpers-pure';
 import { spawnIdeServer } from './ide-server';
@@ -20,7 +30,6 @@ import { displayQuickQuery } from './quick-query';
 import { compileAndRunQueryAgainstDatabase, tmpDirDisposal, UserCancellationException } from './run-queries';
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
-import { promptImportInternetDatabase } from './databaseFetcher';
 
 /**
  * extension.ts
@@ -79,10 +88,12 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
   logger.log('Starting CodeQL extension');
 
   initializeLogging(ctx);
+  languageSupport.install();
 
   const distributionConfigListener = new DistributionConfigListener();
   ctx.subscriptions.push(distributionConfigListener);
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT);
+  const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
+  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
 
   const shouldUpdateOnNextActivationKey = "shouldUpdateOnNextActivation";
 
@@ -181,11 +192,25 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.rawString})`);
+        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
-      case FindDistributionResultKind.IncompatibleDistribution:
-        helpers.showAndLogWarningMessage("The current version of the CodeQL CLI is incompatible with this extension.");
+      case FindDistributionResultKind.IncompatibleDistribution: {
+        const fixGuidanceMessage = (() => {
+          switch (result.distribution.kind) {
+            case DistributionKind.ExtensionManaged:
+              return "Please update the CodeQL CLI by running the \"CodeQL: Check for CLI Updates\" command.";
+            case DistributionKind.CustomPathConfig:
+              return `Please update the \"CodeQL CLI Executable Path\" setting to point to a CLI in the version range ${codeQlVersionRange}.`;
+            case DistributionKind.PathEnvironmentVariable:
+              return `Please update the CodeQL CLI on your PATH to a version compatible with ${codeQlVersionRange}, or ` +
+                `set the \"CodeQL CLI Executable Path\" setting to the path of a CLI version compatible with ${codeQlVersionRange}.`;
+          }
+        })();
+
+        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
+          "is incompatible with this extension. " + fixGuidanceMessage);
         break;
+      }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
         helpers.showAndLogWarningMessage("Compatibility with the configured CodeQL CLI could not be determined. " +
           "You may experience problems using the extension.");
@@ -251,31 +276,39 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
   // of activation.
   errorStubs.forEach(stub => stub.dispose());
 
+  logger.log('Initializing configuration listener...');
   const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(distributionManager);
   ctx.subscriptions.push(qlConfigurationListener);
 
+  logger.log('Initializing CodeQL cli server...');
   const cliServer = new CodeQLCliServer(distributionManager, logger);
   ctx.subscriptions.push(cliServer);
 
+  logger.log('Initializing query server client.');
   const qs = new qsClient.QueryServerClient(qlConfigurationListener, cliServer, {
     logger: queryServerLogger,
   }, task => Window.withProgress({ title: 'CodeQL query server', location: ProgressLocation.Window }, task));
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
+  logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qlConfigurationListener, logger);
   ctx.subscriptions.push(dbm);
+  logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(ctx, cliServer, dbm, qs, getContextStoragePath(ctx));
   ctx.subscriptions.push(databaseUI);
 
+  logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
   const qhm = new QueryHistoryManager(
     ctx,
     queryHistoryConfigurationListener,
     async item => showResultsForCompletedQuery(item, WebviewReveal.Forced)
   );
+  logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
+  logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
   async function showResultsForCompletedQuery(query: CompletedQuery, forceReveal: WebviewReveal): Promise<void> {
@@ -306,6 +339,7 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
 
   ctx.subscriptions.push(tmpDirDisposal);
 
+  logger.log('Initializing CodeQL language server.');
   const client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
     documentSelector: [
       { language: 'ql', scheme: 'file' },
@@ -318,6 +352,7 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     outputChannel: ideServerLogger.outputChannel
   }, true);
 
+  logger.log('Initializing QLTest interface.');
   const testExplorerExtension = extensions.getExtension<TestHub>(testExplorerExtensionId);
   if (testExplorerExtension) {
     const testHub = testExplorerExtension.exports;
@@ -328,6 +363,7 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     ctx.subscriptions.push(testUIService);
   }
 
+  logger.log('Registering top-level command palette commands.');
   ctx.subscriptions.push(commands.registerCommand('codeQL.runQuery', async (uri: Uri | undefined) => await compileAndRunQuery(false, uri)));
   ctx.subscriptions.push(commands.registerCommand('codeQL.quickEval', async (uri: Uri | undefined) => await compileAndRunQuery(true, uri)));
   ctx.subscriptions.push(commands.registerCommand('codeQL.quickQuery', async () => displayQuickQuery(ctx, cliServer, databaseUI)));
@@ -335,20 +371,26 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     await qs.restartQueryServer();
     helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', { outputLogger: queryServerLogger });
   }));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.downloadDatabase', () => promptImportInternetDatabase(dbm, getContextStoragePath(ctx))));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseFolder', () => databaseUI.handleChooseDatabaseFolder()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseArchive', () => databaseUI.handleChooseDatabaseArchive()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseLgtm', () => databaseUI.handleChooseDatabaseLgtm()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseInternet', () => databaseUI.handleChooseDatabaseInternet()));
 
+  logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
-  if (EXPERIMENTAL_FEATURES_SETTING.getValue()) {
-    languages.registerDefinitionProvider(
-      { scheme: archiveFilesystemProvider.zipArchiveScheme },
-      new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
-    );
-    languages.registerReferenceProvider(
-      { scheme: archiveFilesystemProvider.zipArchiveScheme },
-      new TemplateQueryReferenceProvider(cliServer, qs, dbm)
-    );
-  }
+  // Jump-to-definition and find-references
+  logger.log('Registering jump-to-definition handlers.');
+  languages.registerDefinitionProvider(
+    { scheme: archiveFilesystemProvider.zipArchiveScheme },
+    new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
+  );
+  languages.registerReferenceProvider(
+    { scheme: archiveFilesystemProvider.zipArchiveScheme },
+    new TemplateQueryReferenceProvider(cliServer, qs, dbm)
+  );
+
+  logger.log('Successfully finished extension initialization.');
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {

extensions/ql-vscode/src/interface-types.ts

@@ -1,6 +1,6 @@
 import * as sarif from 'sarif';
 import { ResolvableLocationValue } from 'semmle-bqrs';
-import { RawResultSet } from './adapt';
+import { ParsedResultSets } from './adapt';
 
 /**
  * Only ever show this many results per run in interpreted results.
@@ -12,6 +12,11 @@ export const INTERPRETED_RESULTS_PER_RUN_LIMIT = 100;
  */
 export const RAW_RESULTS_LIMIT = 10000;
 
+/**
+ * Show this many rows in a raw result table at a time.
+ */
+export const RAW_RESULTS_PAGE_SIZE = 100;
+
 export interface DatabaseInfo {
   name: string;
   databaseUri: string;
@@ -81,9 +86,10 @@ export interface SetStateMsg {
 
   /**
    * An experimental way of providing results from the extension.
-   * Should be undefined unless config.EXPERIMENTAL_BQRS_SETTING is set to true.
+   * Should be in the WebviewParsedResultSets branch of the type
+   * unless config.EXPERIMENTAL_BQRS_SETTING is set to true.
    */
-  resultSets?: RawResultSet[];
+  parsedResultSets: ParsedResultSets;
 }
 
 /** Advance to the next or previous path no in the path viewer */
@@ -101,7 +107,8 @@ export type FromResultsViewMsg =
   | ToggleDiagnostics
   | ChangeRawResultsSortMsg
   | ChangeInterpretedResultsSortMsg
-  | ResultViewLoaded;
+  | ResultViewLoaded
+  | ChangePage;
 
 interface ViewSourceFileMsg {
   t: 'viewSourceFile';
@@ -122,6 +129,12 @@ interface ResultViewLoaded {
   t: 'resultViewLoaded';
 }
 
+interface ChangePage {
+  t: 'changePage';
+  pageNumber: number; // 0-indexed, displayed to the user as 1-indexed
+  selectedTable: string;
+}
+
 export enum SortDirection {
   asc, desc
 }

extensions/ql-vscode/src/interface-utils.ts

@@ -0,0 +1,22 @@
+import { RawResultSet } from "./adapt";
+import { ResultSetSchema } from "semmle-bqrs";
+import { Interpretation } from "./interface-types";
+
+export const SELECT_TABLE_NAME = '#select';
+export const ALERTS_TABLE_NAME = 'alerts';
+
+export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
+export type PathTableResultSet = { t: 'SarifResultSet'; readonly schema: ResultSetSchema; name: string } & Interpretation;
+
+export type ResultSet =
+  | RawTableResultSet
+  | PathTableResultSet;
+
+export function getDefaultResultSet(resultSets: readonly ResultSet[]): string {
+  return getDefaultResultSetName(resultSets.map(resultSet => resultSet.schema.name));
+}
+
+export function getDefaultResultSetName(resultSetNames: readonly string[]): string {
+  // Choose first available result set from the array
+  return [ALERTS_TABLE_NAME, SELECT_TABLE_NAME, resultSetNames[0]].filter(resultSetName => resultSetNames.includes(resultSetName))[0];
+}

extensions/ql-vscode/src/interface.ts

@@ -10,14 +10,15 @@ import { CodeQLCliServer } from './cli';
 import { DatabaseItem, DatabaseManager } from './databases';
 import { showAndLogErrorMessage } from './helpers';
 import { assertNever } from './helpers-pure';
-import { FromResultsViewMsg, Interpretation, INTERPRETED_RESULTS_PER_RUN_LIMIT, IntoResultsViewMsg, QueryMetadata, ResultsPaths, SortedResultSetInfo, SortedResultsMap, InterpretedResultsSortState, SortDirection } from './interface-types';
+import { FromResultsViewMsg, Interpretation, INTERPRETED_RESULTS_PER_RUN_LIMIT, IntoResultsViewMsg, QueryMetadata, ResultsPaths, SortedResultSetInfo, SortedResultsMap, InterpretedResultsSortState, SortDirection, RAW_RESULTS_PAGE_SIZE } from './interface-types';
 import { Logger } from './logging';
 import * as messages from './messages';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './sarif-utils';
-import { adaptSchema, adaptBqrs, RawResultSet } from './adapt';
+import { adaptSchema, adaptBqrs, RawResultSet, ParsedResultSets } from './adapt';
 import { EXPERIMENTAL_BQRS_SETTING } from './config';
+import { getDefaultResultSetName } from './interface-utils';
 
 /**
  * interface.ts
@@ -115,8 +116,13 @@ function sortInterpretedResults(results: Sarif.Result[], sortState: InterpretedR
   }
 }
 
+function numPagesOfResultSet(resultSet: RawResultSet): number {
+  return Math.ceil(resultSet.schema.tupleCount / RAW_RESULTS_PAGE_SIZE);
+}
+
 export class InterfaceManager extends DisposableObject {
   private _displayedQuery?: CompletedQuery;
+  private _interpretation?: Interpretation;
   private _panel: vscode.WebviewPanel | undefined;
   private _panelLoaded = false;
   private _panelLoadedCallBacks: (() => void)[] = [];
@@ -138,6 +144,7 @@ export class InterfaceManager extends DisposableObject {
         this.handleSelectionChange.bind(this)
       )
     );
+    logger.log('Registering path-step navigation commands.');
     this.push(
       vscode.commands.registerCommand(
         "codeQLQueryResults.nextPathStep",
@@ -287,6 +294,9 @@ export class InterfaceManager extends DisposableObject {
           query.updateInterpretedSortState(this.cliServer, msg.sortState)
         );
         break;
+      case "changePage":
+        await this.showPageOfResults(msg.selectedTable, msg.pageNumber);
+        break;
       default:
         assertNever(msg);
     }
@@ -338,6 +348,7 @@ export class InterfaceManager extends DisposableObject {
     );
 
     this._displayedQuery = results;
+    this._interpretation = interpretation;
 
     const panel = this.getPanel();
     await this.waitForPanelLoaded();
@@ -363,18 +374,37 @@ export class InterfaceManager extends DisposableObject {
       });
     }
 
-    let resultSets: RawResultSet[] | undefined;
+    const getParsedResultSets = async (): Promise<ParsedResultSets> => {
+      if (EXPERIMENTAL_BQRS_SETTING.getValue()) {
+        const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath, RAW_RESULTS_PAGE_SIZE);
 
-    if (EXPERIMENTAL_BQRS_SETTING.getValue()) {
-      resultSets = [];
-      const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath);
-      for (const schema of schemas["result-sets"]) {
-        const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name);
+        const resultSetNames = schemas["result-sets"].map(resultSet => resultSet.name);
+
+        // This may not wind up being the page we actually show, if there are interpreted results,
+        // but speculatively send it anyway.
+        const selectedTable = getDefaultResultSetName(resultSetNames);
+        const schema = schemas["result-sets"].find(resultSet => resultSet.name == selectedTable)!;
+        if (schema === undefined) {
+          return { t: 'WebviewParsed' };
+        }
+
+        const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name, RAW_RESULTS_PAGE_SIZE, schema.pagination?.offsets[0]);
         const adaptedSchema = adaptSchema(schema);
         const resultSet = adaptBqrs(adaptedSchema, chunk);
-        resultSets.push(resultSet);
+
+        return {
+          t: 'ExtensionParsed',
+          pageNumber: 0,
+          numPages: numPagesOfResultSet(resultSet),
+          resultSet,
+          selectedTable: undefined,
+          resultSetNames
+        };
       }
-    }
+      else {
+        return { t: 'WebviewParsed' };
+      }
+    };
 
     await this.postMessage({
       t: "setState",
@@ -383,7 +413,7 @@ export class InterfaceManager extends DisposableObject {
       resultsPath: this.convertPathToWebviewUri(
         results.query.resultsPaths.resultsPath
       ),
-      resultSets,
+      parsedResultSets: await getParsedResultSets(),
       sortedResultsMap,
       database: results.database,
       shouldKeepOldResultsWhileRendering,
@@ -391,6 +421,59 @@ export class InterfaceManager extends DisposableObject {
     });
   }
 
+  /**
+   * Show a page of raw results from the chosen table.
+   */
+  public async showPageOfResults(selectedTable: string, pageNumber: number): Promise<void> {
+    const results = this._displayedQuery;
+    if (results === undefined) {
+      throw new Error('trying to view a page of a query that is not loaded');
+    }
+
+    const sortedResultsMap: SortedResultsMap = {};
+    results.sortedResultsInfo.forEach(
+      (v, k) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(
+          v
+        ))
+    );
+
+    const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath, RAW_RESULTS_PAGE_SIZE);
+
+    const resultSetNames = schemas["result-sets"].map(resultSet => resultSet.name);
+
+    const schema = schemas["result-sets"].find(resultSet => resultSet.name == selectedTable)!;
+    if (schema === undefined)
+      throw new Error(`Query result set '${selectedTable}' not found.`);
+
+    const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name, RAW_RESULTS_PAGE_SIZE, schema.pagination?.offsets[pageNumber]);
+    const adaptedSchema = adaptSchema(schema);
+    const resultSet = adaptBqrs(adaptedSchema, chunk);
+
+    const parsedResultSets: ParsedResultSets = {
+      t: 'ExtensionParsed',
+      pageNumber,
+      resultSet,
+      numPages: numPagesOfResultSet(resultSet),
+      selectedTable: selectedTable,
+      resultSetNames
+    };
+
+    await this.postMessage({
+      t: "setState",
+      interpretation: this._interpretation,
+      origResultsPaths: results.query.resultsPaths,
+      resultsPath: this.convertPathToWebviewUri(
+        results.query.resultsPaths.resultsPath
+      ),
+      parsedResultSets,
+      sortedResultsMap,
+      database: results.database,
+      shouldKeepOldResultsWhileRendering: false,
+      metadata: results.query.metadata
+    });
+  }
+
   private async getTruncatedResults(
     metadata: QueryMetadata | undefined,
     resultsPaths: ResultsPaths,
@@ -401,7 +484,7 @@ export class InterfaceManager extends DisposableObject {
     const sarif = await interpretResults(
       this.cliServer,
       metadata,
-      resultsPaths.resultsPath,
+      resultsPaths,
       sourceInfo
     );
     // For performance reasons, limit the number of results we try
@@ -439,7 +522,7 @@ export class InterfaceManager extends DisposableObject {
   ): Promise<Interpretation | undefined> {
     let interpretation: Interpretation | undefined = undefined;
     if (
-      (await query.hasInterpretedResults()) &&
+      (await query.canHaveInterpretedResults()) &&
       query.quickEvalPosition === undefined // never do results interpretation if quickEval
     ) {
       try {

extensions/ql-vscode/src/languageSupport.ts

@@ -0,0 +1,55 @@
+import { IndentAction, languages } from "vscode";
+
+
+/**
+ * OnEnterRules are available in language-configurations, but you cannot specify them in the language-configuration.json.
+ * They can only be specified programmatically.
+ *
+ * Also, we should keep the language-configuration.json as a json file and register it in the package.json because
+ * it is registered first, before the extension is activated, so language features are available quicker.
+ *
+ * See https://github.com/microsoft/vscode/issues/11514
+ * See https://github.com/microsoft/vscode/blob/master/src/vs/editor/test/common/modes/supports/javascriptOnEnterRules.ts
+ */
+export function install() {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const langConfig = require('../language-configuration.json');
+  // setLanguageConfiguration requires a regexp for the wordpattern, not a string
+  langConfig.wordPattern = new RegExp(langConfig.wordPattern);
+  langConfig.onEnterRules = onEnterRules;
+  langConfig.indentationRules = {
+		decreaseIndentPattern: /^((?!.*?\/\*).*\*\/)?\s*[\}\]].*$/,
+		increaseIndentPattern: /^((?!\/\/).)*(\{[^}"'`]*|\([^)"'`]*|\[[^\]"'`]*)$/
+	};
+
+  languages.setLanguageConfiguration('ql', langConfig);
+  languages.setLanguageConfiguration('qll', langConfig);
+  languages.setLanguageConfiguration('dbscheme', langConfig);
+}
+
+const onEnterRules = [
+  {
+    // e.g. /** | */
+    beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+    afterText: /^\s*\*\/$/,
+    action: { indentAction: IndentAction.IndentOutdent, appendText: ' * ' }
+  }, {
+    // e.g. /** ...|
+    beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+    action: { indentAction: IndentAction.None, appendText: ' * ' }
+  }, {
+    // e.g.  * ...|
+    beforeText: /^(\t|[ ])*[ ]\*([ ]([^\*]|\*(?!\/))*)?$/,
+    oneLineAboveText: /^(\s*(\/\*\*|\*)).*/,
+    action: { indentAction: IndentAction.None, appendText: '* ' }
+  }, {
+    // e.g.  */|
+    beforeText: /^(\t|[ ])*[ ]\*\/\s*$/,
+    action: { indentAction: IndentAction.None, removeText: 1 }
+  },
+  {
+    // e.g.  *-----*/|
+    beforeText: /^(\t|[ ])*[ ]\*[^/]*\*\/\s*$/,
+    action: { indentAction: IndentAction.None, removeText: 1 }
+  }
+];

extensions/ql-vscode/src/query-history.ts

@@ -74,7 +74,7 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
   constructor(private ctx: ExtensionContext) {
   }
 
-  getTreeItem(element: CompletedQuery): vscode.TreeItem {
+  async getTreeItem(element: CompletedQuery): Promise<vscode.TreeItem> {
     const it = new vscode.TreeItem(element.toString());
 
     it.command = {
@@ -83,6 +83,11 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
       arguments: [element],
     };
 
+    // Mark this query history item according to whether it has a
+    // SARIF file so that we can make context menu items conditionally
+    // available.
+    it.contextValue = await element.query.hasInterpretedResults() ? 'interpretedResultsItem' : 'rawResultsItem';
+
     if (!element.didRunSuccessfully) {
       it.iconPath = path.join(this.ctx.extensionPath, FAILED_QUERY_HISTORY_ITEM_ICON);
     }
@@ -218,11 +223,16 @@ export class QueryHistoryManager {
     if (queryHistoryItem.logFileLocation) {
       const uri = vscode.Uri.file(queryHistoryItem.logFileLocation);
       try {
-        await vscode.window.showTextDocument(uri, {
-        });
+        await vscode.window.showTextDocument(uri);
       } catch (e) {
         if (e.message.includes('Files above 50MB cannot be synchronized with extensions')) {
-          const res = await helpers.showBinaryChoiceDialog('File is too large to open in the editor, do you want to open it externally?');
+          const res = await helpers.showBinaryChoiceDialog(
+            `VS Code does not allow extensions to open files >50MB. This file
+exceeds that limit. Do you want to open it outside of VS Code?
+
+You can also try manually opening it inside VS Code by selecting
+the file in the file explorer and dragging it into the workspace.`
+          );
           if (res) {
             try {
               await vscode.commands.executeCommand('revealFileInOS', uri);
@@ -257,6 +267,22 @@ export class QueryHistoryManager {
     }
   }
 
+  async handleViewSarif(queryHistoryItem: CompletedQuery) {
+    try {
+      const hasInterpretedResults = await queryHistoryItem.query.canHaveInterpretedResults();
+      if (hasInterpretedResults) {
+        const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.query.resultsPaths.interpretedResultsPath));
+        await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+      }
+      else {
+        const label = queryHistoryItem.getLabel();
+        helpers.showAndLogInformationMessage(`Query ${label} has no interpreted results.`);
+      }
+    } catch (e) {
+      helpers.showAndLogErrorMessage(e.message);
+    }
+  }
+
   async getQueryText(queryHistoryItem: CompletedQuery): Promise<string> {
     if (queryHistoryItem.options.queryText) {
       return queryHistoryItem.options.queryText;
@@ -290,11 +316,13 @@ export class QueryHistoryManager {
         this.updateTreeViewSelectionIfVisible();
       }
     });
+    logger.log('Registering query history panel commands.');
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.openQuery', this.handleOpenQuery));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.removeHistoryItem', this.handleRemoveHistoryItem.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.setLabel', this.handleSetLabel.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryLog', this.handleShowQueryLog.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryText', this.handleShowQueryText.bind(this)));
+    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.viewSarif', this.handleViewSarif.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.itemClicked', async (item) => {
       return this.handleItemClicked(item);
     }));

extensions/ql-vscode/src/query-results.ts

@@ -5,7 +5,7 @@ import * as cli from './cli';
 import * as sarif from 'sarif';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState } from "./interface-types";
+import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState, ResultsPaths } from "./interface-types";
 import { QueryHistoryConfig } from "./config";
 import { QueryHistoryItemOptions } from "./query-history";
 
@@ -54,13 +54,6 @@ export class CompletedQuery implements QueryWithResults {
     return helpers.getQueryName(this.query);
   }
 
-  /**
-   * Holds if this query should produce interpreted results.
-   */
-  canInterpretedResults(): Promise<boolean> {
-    return this.query.dbItem.hasMetadataFile();
-  }
-
   get statusString(): string {
     switch (this.result.resultType) {
       case messages.QueryResultType.CANCELLATION:
@@ -130,9 +123,8 @@ export class CompletedQuery implements QueryWithResults {
 /**
  * Call cli command to interpret results.
  */
-export async function interpretResults(server: cli.CodeQLCliServer, metadata: QueryMetadata | undefined, resultsPath: string, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
-  const interpretedResultsPath = resultsPath + ".interpreted.sarif";
-
+export async function interpretResults(server: cli.CodeQLCliServer, metadata: QueryMetadata | undefined, resultsPaths: ResultsPaths, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
+  const { resultsPath, interpretedResultsPath } = resultsPaths;
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }

extensions/ql-vscode/src/run-queries.ts

@@ -157,15 +157,22 @@ export class QueryInfo {
   }
 
   /**
-   * Holds if this query should produce interpreted results.
+   * Holds if this query can in principle produce interpreted results.
    */
-  async hasInterpretedResults(): Promise<boolean> {
+  async canHaveInterpretedResults(): Promise<boolean> {
     const hasMetadataFile = await this.dbItem.hasMetadataFile();
     if (!hasMetadataFile) {
       logger.log("Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.");
     }
     return hasMetadataFile;
   }
+
+  /**
+   * Holds if this query actually has produced interpreted results.
+   */
+  async hasInterpretedResults(): Promise<boolean> {
+    return fs.pathExists(this.resultsPaths.interpretedResultsPath);
+  }
 }
 
 export interface QueryWithResults {

extensions/ql-vscode/src/test-ui.ts

@@ -5,6 +5,7 @@ import { TestTreeNode } from './test-tree-node';
 import { DisposableObject, UIService } from 'semmle-vscode-utils';
 import { TestHub, TestController, TestAdapter, TestRunStartedEvent, TestRunFinishedEvent, TestEvent, TestSuiteEvent } from 'vscode-test-adapter-api';
 import { QLTestAdapter, getExpectedFile, getActualFile } from './test-adapter';
+import { logger } from './logging';
 
 type VSCodeTestEvent = TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent | TestEvent;
 
@@ -32,6 +33,7 @@ export class TestUIService extends UIService implements TestController {
   constructor(private readonly testHub: TestHub) {
     super();
 
+    logger.log('Registering CodeQL test panel commands.');
     this.registerCommand('codeQLTests.showOutputDifferences', this.showOutputDifferences);
     this.registerCommand('codeQLTests.acceptOutput', this.acceptOutput);
 

extensions/ql-vscode/src/view/alert-table.tsx

@@ -5,9 +5,10 @@ import * as Keys from '../result-keys';
 import { LocationStyle } from 'semmle-bqrs';
 import * as octicons from './octicons';
 import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection } from './result-table-utils';
-import { PathTableResultSet, onNavigation, NavigationEvent, vscode } from './results';
+import { onNavigation, NavigationEvent, vscode } from './results';
 import { parseSarifPlainTextMessage, parseSarifLocation } from '../sarif-utils';
 import { InterpretedResultsSortColumn, SortDirection, InterpretedResultsSortState } from '../interface-types';
+import { PathTableResultSet } from '../interface-utils';
 
 export type PathTableProps = ResultTableProps & { resultSet: PathTableResultSet };
 export interface PathTableState {

extensions/ql-vscode/src/view/raw-results-table.tsx

@@ -1,12 +1,14 @@
 import * as React from "react";
 import { renderLocation, ResultTableProps, zebraStripe, className, nextSortDirection } from "./result-table-utils";
-import { RawTableResultSet, vscode } from "./results";
+import { vscode } from "./results";
 import { ResultValue } from "../adapt";
 import { SortDirection, RAW_RESULTS_LIMIT, RawResultsSortState } from "../interface-types";
+import { RawTableResultSet } from "../interface-utils";
 
 export type RawTableProps = ResultTableProps & {
   resultSet: RawTableResultSet;
   sortState?: RawResultsSortState;
+  offset: number;
 };
 
 export class RawTable extends React.Component<RawTableProps, {}> {
@@ -28,7 +30,7 @@ export class RawTable extends React.Component<RawTableProps, {}> {
       <tr key={rowIndex} {...zebraStripe(rowIndex)}>
         {
           [
-            <td key={-1}>{rowIndex + 1}</td>,
+            <td key={-1}>{rowIndex + 1 + this.props.offset}</td>,
             ...row.map((value, columnIndex) =>
               <td key={columnIndex}>
                 {

extensions/ql-vscode/src/view/result-table-utils.tsx

@@ -1,8 +1,9 @@
 import * as React from 'react';
 import { LocationValue, ResolvableLocationValue, tryGetResolvableLocation } from 'semmle-bqrs';
 import { RawResultsSortState, QueryMetadata, SortDirection } from '../interface-types';
-import { ResultSet, vscode } from './results';
+import { vscode } from './results';
 import { assertNever } from '../helpers-pure';
+import { ResultSet } from '../interface-utils';
 
 export interface ResultTableProps {
   resultSet: ResultSet;
@@ -10,6 +11,7 @@ export interface ResultTableProps {
   metadata?: QueryMetadata;
   resultsPath: string | undefined;
   sortState?: RawResultsSortState;
+  offset: number;
 
   /**
    * Holds if there are any raw results. When that is the case, we

extensions/ql-vscode/src/view/result-tables.tsx

@@ -1,14 +1,17 @@
 import * as React from 'react';
-import { DatabaseInfo, Interpretation, RawResultsSortState, QueryMetadata, ResultsPaths, InterpretedResultsSortState } from '../interface-types';
+import { DatabaseInfo, Interpretation, RawResultsSortState, QueryMetadata, ResultsPaths, InterpretedResultsSortState, RAW_RESULTS_PAGE_SIZE } from '../interface-types';
 import { PathTable } from './alert-table';
 import { RawTable } from './raw-results-table';
 import { ResultTableProps, tableSelectionHeaderClassName, toggleDiagnosticsClassName, alertExtrasClassName } from './result-table-utils';
-import { ResultSet, vscode } from './results';
+import { vscode } from './results';
+import { ParsedResultSets, ExtensionParsedResultSets } from '../adapt';
+import { ResultSet, ALERTS_TABLE_NAME, SELECT_TABLE_NAME, getDefaultResultSet } from '../interface-utils';
 
 /**
  * Properties for the `ResultTables` component.
  */
 export interface ResultTablesProps {
+  parsedResultSets: ParsedResultSets;
   rawResultSets: readonly ResultSet[];
   interpretation: Interpretation | undefined;
   database: DatabaseInfo;
@@ -25,10 +28,9 @@ export interface ResultTablesProps {
  */
 interface ResultTablesState {
   selectedTable: string; // name of selected result set
+  selectedPage: string; // stringified selected page
 }
 
-const ALERTS_TABLE_NAME = 'alerts';
-const SELECT_TABLE_NAME = '#select';
 const UPDATING_RESULTS_TEXT_CLASS_NAME = "vscode-codeql__result-tables-updating-text";
 
 function getResultCount(resultSet: ResultSet): number {
@@ -75,23 +77,66 @@ export class ResultTables
     return resultSets;
   }
 
+  private getResultSetNames(resultSets: ResultSet[]): string[] {
+    if (this.props.parsedResultSets.t === 'ExtensionParsed') {
+      return this.props.parsedResultSets.resultSetNames.concat([ALERTS_TABLE_NAME]);
+    }
+    else {
+      return resultSets.map(resultSet => resultSet.schema.name);
+    }
+  }
+
+  /**
+   * Holds if we have a result set obtained from the extension that came
+   * from the ExtensionParsed branch of ParsedResultSets. This is evidence
+   * that the user has the experimental flag turned on that allows extension-side
+   * bqrs parsing.
+   */
+  paginationAllowed(): boolean {
+    return this.props.parsedResultSets.t === 'ExtensionParsed';
+  }
+
+  /**
+   * Holds if we actually should show pagination interface right now. This is
+   * still false for the time being when we're viewing alerts.
+   */
+  paginationEnabled(): boolean {
+    return this.paginationAllowed() &&
+      this.props.parsedResultSets.selectedTable !== ALERTS_TABLE_NAME &&
+      this.state.selectedTable !== ALERTS_TABLE_NAME;
+  }
+
   constructor(props: ResultTablesProps) {
     super(props);
 
-    this.state = {
-      // Get the result set that should be displayed by default
-      selectedTable: ResultTables.getDefaultResultSet(this.getResultSets())
-    };
-  }
+    const selectedTable = props.parsedResultSets.selectedTable || getDefaultResultSet(this.getResultSets());
 
-  private static getDefaultResultSet(resultSets: readonly ResultSet[]): string {
-    const resultSetNames = resultSets.map(resultSet => resultSet.schema.name);
-    // Choose first available result set from the array
-    return [ALERTS_TABLE_NAME, SELECT_TABLE_NAME, resultSets[0].schema.name].filter(resultSetName => resultSetNames.includes(resultSetName))[0];
+    let selectedPage: string;
+    switch (props.parsedResultSets.t) {
+      case 'ExtensionParsed':
+        selectedPage = (props.parsedResultSets.pageNumber + 1) + '';
+        break;
+      case 'WebviewParsed':
+        selectedPage = '';
+        break;
+    }
+
+    this.state = { selectedTable, selectedPage };
   }
 
   private onTableSelectionChange = (event: React.ChangeEvent<HTMLSelectElement>): void => {
-    this.setState({ selectedTable: event.target.value });
+    const selectedTable = event.target.value;
+    const fetchPageFromExtension = this.paginationAllowed() && selectedTable !== ALERTS_TABLE_NAME;
+
+    if (fetchPageFromExtension) {
+      vscode.postMessage({
+        t: 'changePage',
+        pageNumber: 0,
+        selectedTable
+      });
+    }
+    else
+      this.setState({ selectedTable });
   }
 
   private alertTableExtras(): JSX.Element | undefined {
@@ -118,24 +163,81 @@ export class ResultTables
     </div>;
   }
 
+  getOffset(): number {
+    const { parsedResultSets } = this.props;
+    switch (parsedResultSets.t) {
+      case 'ExtensionParsed':
+        return parsedResultSets.pageNumber * RAW_RESULTS_PAGE_SIZE;
+      case 'WebviewParsed':
+        return 0;
+    }
+  }
+
+  renderPageButtons(resultSets: ExtensionParsedResultSets): JSX.Element {
+    const selectedTable = this.state.selectedTable;
+    const onChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+      this.setState({ selectedPage: e.target.value });
+    };
+    const choosePage = (input: string) => {
+      const pageNumber = parseInt(input);
+      if (pageNumber !== undefined && !isNaN(pageNumber)) {
+        const actualPageNumber = Math.max(0, Math.min(pageNumber - 1, resultSets.numPages - 1));
+        vscode.postMessage({
+          t: 'changePage',
+          pageNumber: actualPageNumber,
+          selectedTable,
+        });
+      }
+    };
+
+    const prevPage = (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => {
+      vscode.postMessage({
+        t: 'changePage',
+        pageNumber: Math.max(resultSets.pageNumber - 1, 0),
+        selectedTable,
+      });
+    };
+    const nextPage = (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => {
+      vscode.postMessage({
+        t: 'changePage',
+        pageNumber: Math.min(resultSets.pageNumber + 1, resultSets.numPages - 1),
+        selectedTable,
+      });
+    };
+    return <span>
+      <button onClick={prevPage} >&lt;</button>
+      <input value={this.state.selectedPage} onChange={onChange}
+        onBlur={e => choosePage(e.target.value)}
+        onKeyDown={e => { if (e.keyCode === 13) choosePage((e.target as HTMLInputElement).value); }}
+      />
+      <button value=">" onClick={nextPage} >&gt;</button>
+    </span>;
+  }
+
+  renderButtons(): JSX.Element {
+    if (this.props.parsedResultSets.t === 'ExtensionParsed' && this.paginationEnabled())
+      return this.renderPageButtons(this.props.parsedResultSets);
+    else
+      return <span />;
+  }
+
   render(): React.ReactNode {
     const { selectedTable } = this.state;
     const resultSets = this.getResultSets();
+    const resultSetNames = this.getResultSetNames(resultSets);
 
     const resultSet = resultSets.find(resultSet => resultSet.schema.name == selectedTable);
     const nonemptyRawResults = resultSets.some(resultSet => resultSet.t == 'RawResultSet' && resultSet.rows.length > 0);
     const numberOfResults = resultSet && renderResultCountString(resultSet);
 
+    const resultSetOptions =
+      resultSetNames.map(name => <option key={name} value={name}>{name}</option>);
+
     return <div>
+      {this.renderButtons()}
       <div className={tableSelectionHeaderClassName}>
         <select value={selectedTable} onChange={this.onTableSelectionChange}>
-          {
-            resultSets.map(resultSet =>
-              <option key={resultSet.schema.name} value={resultSet.schema.name}>
-                {resultSet.schema.name}
-              </option>
-            )
-          }
+          {resultSetOptions}
         </select>
         {numberOfResults}
         {selectedTable === ALERTS_TABLE_NAME ? this.alertTableExtras() : undefined}
@@ -152,7 +254,8 @@ export class ResultTables
           resultsPath={this.props.resultsPath}
           sortState={this.props.sortStates.get(resultSet.schema.name)}
           nonemptyRawResults={nonemptyRawResults}
-          showRawResults={() => { this.setState({ selectedTable: SELECT_TABLE_NAME }); }} />
+          showRawResults={() => { this.setState({ selectedTable: SELECT_TABLE_NAME }); }}
+          offset={this.getOffset()} />
       }
     </div>;
   }

extensions/ql-vscode/src/view/results.tsx

@@ -1,12 +1,13 @@
 import * as React from 'react';
 import * as Rdom from 'react-dom';
 import * as bqrs from 'semmle-bqrs';
-import { ElementBase, PrimitiveColumnValue, PrimitiveTypeKind, ResultSetSchema, tryGetResolvableLocation } from 'semmle-bqrs';
+import { ElementBase, PrimitiveColumnValue, PrimitiveTypeKind, tryGetResolvableLocation } from 'semmle-bqrs';
 import { assertNever } from '../helpers-pure';
 import { DatabaseInfo, FromResultsViewMsg, Interpretation, IntoResultsViewMsg, SortedResultSetInfo, RawResultsSortState, NavigatePathMsg, QueryMetadata, ResultsPaths } from '../interface-types';
 import { EventHandlers as EventHandlerList } from './event-handler-list';
 import { ResultTables } from './result-tables';
-import { RawResultSet, ResultValue, ResultRow } from '../adapt';
+import { ResultValue, ResultRow, ParsedResultSets } from '../adapt';
+import { ResultSet } from '../interface-utils';
 
 /**
  * results.tsx
@@ -24,13 +25,6 @@ interface VsCodeApi {
 declare const acquireVsCodeApi: () => VsCodeApi;
 export const vscode = acquireVsCodeApi();
 
-export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
-export type PathTableResultSet = { t: 'SarifResultSet'; readonly schema: ResultSetSchema; name: string } & Interpretation;
-
-export type ResultSet =
-  | RawTableResultSet
-  | PathTableResultSet;
-
 async function* getChunkIterator(response: Response): AsyncIterableIterator<Uint8Array> {
   if (!response.ok) {
     throw new Error(`Failed to load results: (${response.status}) ${response.statusText}`);
@@ -107,8 +101,8 @@ async function parseResultSets(response: Response): Promise<readonly ResultSet[]
 }
 
 interface ResultsInfo {
+  parsedResultSets: ParsedResultSets;
   resultsPath: string;
-  resultSets: ResultSet[] | undefined;
   origResultsPaths: ResultsPaths;
   database: DatabaseInfo;
   interpretation: Interpretation | undefined;
@@ -169,7 +163,7 @@ class App extends React.Component<{}, ResultsViewState> {
       case 'setState':
         this.updateStateWithNewResultsInfo({
           resultsPath: msg.resultsPath,
-          resultSets: msg.resultSets?.map(x => ({ t: 'RawResultSet', ...x })),
+          parsedResultSets: msg.parsedResultSets,
           origResultsPaths: msg.origResultsPaths,
           sortedResultsMap: new Map(Object.entries(msg.sortedResultsMap)),
           database: msg.database,
@@ -221,6 +215,16 @@ class App extends React.Component<{}, ResultsViewState> {
     });
   }
 
+  private async getResultSets(resultsInfo: ResultsInfo): Promise<readonly ResultSet[]> {
+    const parsedResultSets = resultsInfo.parsedResultSets;
+    switch (parsedResultSets.t) {
+      case 'WebviewParsed': return await this.fetchResultSets(resultsInfo);
+      case 'ExtensionParsed': {
+        return [{ t: 'RawResultSet', ...parsedResultSets.resultSet }];
+      }
+    }
+  }
+
   private async loadResults(): Promise<void> {
     const resultsInfo = this.state.nextResultsInfo;
     if (resultsInfo === null) {
@@ -230,7 +234,7 @@ class App extends React.Component<{}, ResultsViewState> {
     let results: Results | null = null;
     let statusText = '';
     try {
-      const resultSets = resultsInfo.resultSets || await this.getResultSets(resultsInfo);
+      const resultSets = await this.getResultSets(resultsInfo);
       results = {
         resultSets,
         database: resultsInfo.database,
@@ -265,7 +269,11 @@ class App extends React.Component<{}, ResultsViewState> {
     });
   }
 
-  private async getResultSets(resultsInfo: ResultsInfo): Promise<readonly ResultSet[]> {
+  /**
+   * This is deprecated, because it calls `fetch`. We are moving
+   * towards doing all bqrs parsing in the extension.
+   */
+  private async fetchResultSets(resultsInfo: ResultsInfo): Promise<readonly ResultSet[]> {
     const unsortedResponse = await fetch(resultsInfo.resultsPath);
     const unsortedResultSets = await parseResultSets(unsortedResponse);
     return Promise.all(unsortedResultSets.map(async unsortedResultSet => {
@@ -291,7 +299,10 @@ class App extends React.Component<{}, ResultsViewState> {
   render(): JSX.Element {
     const displayedResults = this.state.displayedResults;
     if (displayedResults.results !== null && displayedResults.resultsInfo !== null) {
-      return <ResultTables rawResultSets={displayedResults.results.resultSets}
+      const parsedResultSets = displayedResults.resultsInfo.parsedResultSets;
+      return <ResultTables
+        parsedResultSets={parsedResultSets}
+        rawResultSets={displayedResults.results.resultSets}
         interpretation={displayedResults.resultsInfo ? displayedResults.resultsInfo.interpretation : undefined}
         database={displayedResults.results.database}
         origResultsPaths={displayedResults.resultsInfo.origResultsPaths}

extensions/ql-vscode/src/vscode-tests/no-workspace/archive-filesystem-provider.test.ts

@@ -1,8 +1,10 @@
-import { expect } from "chai";
-import * as path from "path";
-import { ArchiveFileSystemProvider, decodeSourceArchiveUri, encodeSourceArchiveUri, ZipFileReference } from "../../archive-filesystem-provider";
+import { expect } from 'chai';
+import * as path from 'path';
 
-describe("archive filesystem provider", () => {
+import { encodeSourceArchiveUri, ArchiveFileSystemProvider, decodeSourceArchiveUri, ZipFileReference } from '../../archive-filesystem-provider';
+import { FileType, FileSystemError } from 'vscode';
+
+describe('archive-filesystem-provider', () => {
   it("reads empty file correctly", async () => {
     const archiveProvider = new ArchiveFileSystemProvider();
     const uri = encodeSourceArchiveUri({
@@ -12,6 +14,98 @@ describe("archive filesystem provider", () => {
     const data = await archiveProvider.readFile(uri);
     expect(data.length).to.equal(0);
   });
+
+  it("read non-empty file correctly", async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/textFile.txt"
+    });
+    const data = await archiveProvider.readFile(uri);
+    expect(Buffer.from(data).toString('utf8')).to.be.equal('I am a text\n');
+  });
+
+  it("read a directory", async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1"
+    });
+    const files = await archiveProvider.readDirectory(uri);
+    expect(files).to.be.deep.equal([
+      ['folder2', FileType.Directory],
+      ['textFile.txt', FileType.File],
+      ['textFile2.txt', FileType.File],
+    ]);
+  });
+
+  it('should handle a missing directory', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/not-here"
+    });
+    try {
+      await archiveProvider.readDirectory(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it('should handle a missing file', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/not-here"
+    });
+    try {
+      await archiveProvider.readFile(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it('should handle reading a file as a directory', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/textFile.txt"
+    });
+    try {
+      await archiveProvider.readDirectory(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it('should handle reading a directory as a file', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/folder2"
+    });
+    try {
+      await archiveProvider.readFile(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it("read a nested directory", async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/folder2"
+    });
+    const files = await archiveProvider.readDirectory(uri);
+    expect(files).to.be.deep.equal([
+      ['textFile3.txt', FileType.File],
+    ]);
+  });
 });
 
 describe('source archive uri encoding', function() {

extensions/ql-vscode/src/vscode-tests/no-workspace/cli-version.test.ts

@@ -1,50 +0,0 @@
-import { expect } from "chai";
-import "mocha";
-import { tryParseVersionString } from "../../cli-version";
-
-describe("Version parsing", () => {
-  it("should accept version without prerelease and build metadata", () => {
-    const v = tryParseVersionString("3.2.4")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.be.undefined;
-    expect(v.buildMetadata).to.be.undefined;
-  });
-
-  it("should accept v at the beginning of the version", () => {
-    const v = tryParseVersionString("v3.2.4")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.be.undefined;
-    expect(v.buildMetadata).to.be.undefined;
-  });
-
-  it("should accept version with prerelease", () => {
-    const v = tryParseVersionString("v3.2.4-alpha.0")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.equal("alpha.0");
-    expect(v.buildMetadata).to.be.undefined;
-  });
-
-  it("should accept version with prerelease and build metadata", () => {
-    const v = tryParseVersionString("v3.2.4-alpha.0+abcdef0")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.equal("alpha.0");
-    expect(v.buildMetadata).to.equal("abcdef0");
-  });
-
-  it("should accept version with build metadata", () => {
-    const v = tryParseVersionString("v3.2.4+abcdef0")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.be.undefined;
-    expect(v.buildMetadata).to.equal("abcdef0");
-  });
-});

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -0,0 +1,140 @@
+import "vscode-test";
+import "mocha";
+import * as chaiAsPromised from "chai-as-promised";
+import * as sinon from "sinon";
+// import * as sinonChai from 'sinon-chai';
+import * as path from "path";
+import * as fs from "fs-extra";
+import * as tmp from "tmp";
+import * as chai from "chai";
+import { window } from "vscode";
+
+import {
+  convertToDatabaseUrl,
+  looksLikeLgtmUrl,
+  findDirWithFile,
+} from "../../databaseFetcher";
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
+describe("databaseFetcher", () => {
+  describe("convertToDatabaseUrl", () => {
+    let quickPickSpy: sinon.SinonStub;
+    beforeEach(() => {
+      quickPickSpy = sinon.stub(window, "showQuickPick");
+    });
+
+    afterEach(() => {
+      (window.showQuickPick as sinon.SinonStub).restore();
+    });
+
+    it("should convert a project url to a database url", async () => {
+      quickPickSpy.returns("javascript" as any);
+      const lgtmUrl = "https://lgtm.com/projects/g/github/codeql";
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+
+      expect(dbUrl).to.equal(
+        "https://lgtm.com/api/v1.0/snapshots/1506465042581/javascript"
+      );
+      expect(quickPickSpy.firstCall.args[0]).to.contain("javascript");
+      expect(quickPickSpy.firstCall.args[0]).to.contain("python");
+    });
+
+    it("should convert a project url to a database url with extra path segments", async () => {
+      quickPickSpy.returns("python" as any);
+      const lgtmUrl =
+        "https://lgtm.com/projects/g/github/codeql/subpage/subpage2?query=xxx";
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+
+      expect(dbUrl).to.equal(
+        "https://lgtm.com/api/v1.0/snapshots/1506465042581/python"
+      );
+    });
+
+    it("should fail on a nonexistant prohect", async () => {
+      quickPickSpy.returns("javascript" as any);
+      const lgtmUrl = "https://lgtm.com/projects/g/github/hucairz";
+      expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
+    });
+  });
+
+  describe("looksLikeLgtmUrl", () => {
+    it("should handle invalid urls", () => {
+      expect(looksLikeLgtmUrl("")).to.be.false;
+      expect(looksLikeLgtmUrl("http://lgtm.com/projects/g/github/codeql")).to.be
+        .false;
+      expect(looksLikeLgtmUrl("https://ww.lgtm.com/projects/g/github/codeql"))
+        .to.be.false;
+      expect(looksLikeLgtmUrl("https://ww.lgtm.com/projects/g/github")).to.be
+        .false;
+    });
+
+    it("should handle valid urls", () => {
+      expect(looksLikeLgtmUrl("https://lgtm.com/projects/g/github/codeql")).to
+        .be.true;
+      expect(looksLikeLgtmUrl("https://www.lgtm.com/projects/g/github/codeql"))
+        .to.be.true;
+      expect(
+        looksLikeLgtmUrl("https://lgtm.com/projects/g/github/codeql/sub/pages")
+      ).to.be.true;
+      expect(
+        looksLikeLgtmUrl(
+          "https://lgtm.com/projects/g/github/codeql/sub/pages?query=string"
+        )
+      ).to.be.true;
+    });
+  });
+
+  describe("findDirWithFile", () => {
+    let dir: tmp.DirResult;
+    beforeEach(() => {
+      dir = tmp.dirSync({ unsafeCleanup: true });
+      createFile("a");
+      createFile("b");
+      createFile("c");
+
+      createDir("dir1");
+      createFile("dir1", "d");
+      createFile("dir1", "e");
+      createFile("dir1", "f");
+
+      createDir("dir2");
+      createFile("dir2", "g");
+      createFile("dir2", "h");
+      createFile("dir2", "i");
+
+      createDir("dir2", "dir3");
+      createFile("dir2", "dir3", "j");
+      createFile("dir2", "dir3", "k");
+      createFile("dir2", "dir3", "l");
+    });
+
+    it("should find files", async () => {
+      expect(await findDirWithFile(dir.name, "k")).to.equal(
+        path.join(dir.name, "dir2", "dir3")
+      );
+      expect(await findDirWithFile(dir.name, "h")).to.equal(
+        path.join(dir.name, "dir2")
+      );
+      expect(await findDirWithFile(dir.name, "z", "a")).to.equal(dir.name);
+      // there's some slight indeterminism when more than one name exists
+      // but in general, this will find files in the current directory before
+      // finding files in sub-dirs
+      expect(await findDirWithFile(dir.name, "k", "a")).to.equal(dir.name);
+    });
+
+
+    it("should not find files", async () => {
+      expect(await findDirWithFile(dir.name, "x", "y", "z")).to.be.undefined;
+    });
+
+
+    function createFile(...segments: string[]) {
+      fs.createFileSync(path.join(dir.name, ...segments));
+    }
+
+    function createDir(...segments: string[]) {
+      fs.mkdirSync(path.join(dir.name, ...segments));
+    }
+  });
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts

@@ -0,0 +1,47 @@
+import 'vscode-test';
+import 'mocha';
+import * as tmp from 'tmp';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { expect } from 'chai';
+import { Uri } from 'vscode';
+
+import { DatabaseUI } from '../../databases-ui';
+
+describe('databases-ui', () => {
+  describe('fixDbUri', () => {
+    const fixDbUri = (DatabaseUI.prototype as any).fixDbUri;
+    it('should choose current directory direcory normally', async () => {
+      const dir = tmp.dirSync().name;
+      const uri = await fixDbUri(Uri.file(dir));
+      expect(uri.toString()).to.eq(Uri.file(dir).toString());
+    });
+
+    it('should choose parent direcory when file is selected', async () => {
+      const file = tmp.fileSync().name;
+      const uri = await fixDbUri(Uri.file(file));
+      expect(uri.toString()).to.eq(Uri.file(path.dirname(file)).toString());
+    });
+
+    it('should choose parent direcory when db-* is selected', async () => {
+      const dir = tmp.dirSync().name;
+      const dbDir = path.join(dir, 'db-hucairz');
+      await fs.mkdirs(dbDir);
+
+      const uri = await fixDbUri(Uri.file(dbDir));
+      expect(uri.toString()).to.eq(Uri.file(dir).toString());
+    });
+
+    it('should choose parent\'s parent direcory when file selected is in db-*', async () => {
+      const dir = tmp.dirSync().name;
+      const dbDir = path.join(dir, 'db-hucairz');
+      const file = path.join(dbDir, 'nested');
+      await fs.mkdirs(dbDir);
+      await fs.createFile(file);
+
+      const uri = await fixDbUri(Uri.file(file));
+      expect(uri.toString()).to.eq(Uri.file(dir).toString());
+    });
+  });
+
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -1,14 +1,14 @@
 import * as chai from "chai";
 import * as path from "path";
 import * as fetch from "node-fetch";
-import 'chai/register-should';
-import * as sinonChai from 'sinon-chai';
-import * as sinon from 'sinon';
+import "chai/register-should";
+import * as semver from "semver";
+import * as sinonChai from "sinon-chai";
+import * as sinon from "sinon";
 import * as pq from "proxyquire";
 import "mocha";
 
-import { Version } from "../../cli-version";
-import { GithubRelease, GithubReleaseAsset, ReleasesApiConsumer, versionCompare } from "../../distribution";
+import { GithubRelease, GithubReleaseAsset, ReleasesApiConsumer } from "../../distribution";
 
 const proxyquire = pq.noPreserveCache();
 chai.use(sinonChai);
@@ -17,46 +17,58 @@ const expect = chai.expect;
 describe("Releases API consumer", () => {
   const owner = "someowner";
   const repo = "somerepo";
-  const sampleReleaseResponse: GithubRelease[] = [
-    {
-      "assets": [],
-      "created_at": "2019-09-01T00:00:00Z",
-      "id": 1,
-      "name": "",
-      "prerelease": false,
-      "tag_name": "v2.1.0"
-    },
-    {
-      "assets": [],
-      "created_at": "2019-08-10T00:00:00Z",
-      "id": 2,
-      "name": "",
-      "prerelease": false,
-      "tag_name": "v3.1.1"
-    },
-    {
-      "assets": [],
-      "created_at": "2019-09-05T00:00:00Z",
-      "id": 3,
-      "name": "",
-      "prerelease": false,
-      "tag_name": "v2.0.0"
-    },
-    {
-      "assets": [],
-      "created_at": "2019-08-11T00:00:00Z",
-      "id": 4,
-      "name": "",
-      "prerelease": true,
-      "tag_name": "v3.1.2-pre"
-    },
-  ];
-  const unconstrainedVersionConstraint = {
-    description: "*",
-    isVersionCompatible: () => true
-  };
+  const unconstrainedVersionRange = new semver.Range("*");
+
+  describe("picking the latest release", () => {
+    const sampleReleaseResponse: GithubRelease[] = [
+      {
+        "assets": [],
+        "created_at": "2019-09-01T00:00:00Z",
+        "id": 1,
+        "name": "",
+        "prerelease": false,
+        "tag_name": "v2.1.0"
+      },
+      {
+        "assets": [],
+        "created_at": "2019-08-10T00:00:00Z",
+        "id": 2,
+        "name": "",
+        "prerelease": false,
+        "tag_name": "v3.1.1"
+      },
+      {
+        "assets": [{
+          id: 1,
+          name: "exampleAsset.txt",
+          size: 1
+        }],
+        "created_at": "2019-09-05T00:00:00Z",
+        "id": 3,
+        "name": "",
+        "prerelease": false,
+        "tag_name": "v2.0.0"
+      },
+      {
+        "assets": [],
+        "created_at": "2019-08-11T00:00:00Z",
+        "id": 4,
+        "name": "",
+        "prerelease": true,
+        "tag_name": "v3.1.2-pre-1.1"
+      },
+      // Release ID 5 is older than release ID 4 but its version has a higher precedence, so release
+      // ID 5 should be picked over release ID 4.
+      {
+        "assets": [],
+        "created_at": "2019-08-09T00:00:00Z",
+        "id": 5,
+        "name": "",
+        "prerelease": true,
+        "tag_name": "v3.1.2-pre-2.0"
+      },
+    ];
 
-  it("picking latest release: is based on version", async () => {
     class MockReleasesApiConsumer extends ReleasesApiConsumer {
       protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
         if (apiPath === `/repos/${owner}/${repo}/releases`) {
@@ -66,45 +78,55 @@ describe("Releases API consumer", () => {
       }
     }
 
-    const consumer = new MockReleasesApiConsumer(owner, repo);
+    it("picked release has version with the highest precedence", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
 
-    const latestRelease = await consumer.getLatestRelease(unconstrainedVersionConstraint);
-    expect(latestRelease.id).to.equal(2);
-  });
-
-  it("picking latest release: obeys version constraints", async () => {
-    class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
-        if (apiPath === `/repos/${owner}/${repo}/releases`) {
-          return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
-        }
-        return Promise.reject(new Error(`Unknown API path: ${apiPath}`));
-      }
-    }
-
-    const consumer = new MockReleasesApiConsumer(owner, repo);
-
-    const latestRelease = await consumer.getLatestRelease({
-      description: "2.*.*",
-      isVersionCompatible: version => version.majorVersion === 2
+      const latestRelease = await consumer.getLatestRelease(unconstrainedVersionRange);
+      expect(latestRelease.id).to.equal(2);
     });
-    expect(latestRelease.id).to.equal(1);
-  });
 
-  it("picking latest release: includes prereleases when option set", async () => {
-    class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
-        if (apiPath === `/repos/${owner}/${repo}/releases`) {
-          return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
-        }
-        return Promise.reject(new Error(`Unknown API path: ${apiPath}`));
-      }
-    }
+    it("version of picked release is within the version range", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
 
-    const consumer = new MockReleasesApiConsumer(owner, repo);
+      const latestRelease = await consumer.getLatestRelease(new semver.Range("2.*.*"));
+      expect(latestRelease.id).to.equal(1);
+    });
 
-    const latestRelease = await consumer.getLatestRelease(unconstrainedVersionConstraint, true);
-    expect(latestRelease.id).to.equal(4);
+    it("fails if none of the releases are within the version range", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      await chai.expect(
+        consumer.getLatestRelease(new semver.Range("5.*.*"))
+      ).to.be.rejectedWith(Error);
+    });
+
+    it("picked release passes additional compatibility test if an additional compatibility test is specified", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      const latestRelease = await consumer.getLatestRelease(
+        new semver.Range("2.*.*"),
+        true,
+        release => release.assets.some(asset => asset.name === "exampleAsset.txt")
+      );
+      expect(latestRelease.id).to.equal(3);
+    });
+
+    it("fails if none of the releases pass the additional compatibility test", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      await chai.expect(consumer.getLatestRelease(
+        new semver.Range("2.*.*"),
+        true,
+        release => release.assets.some(asset => asset.name === "otherExampleAsset.txt")
+      )).to.be.rejectedWith(Error);
+    });
+
+    it("picked release is the most recent prerelease when includePrereleases is set", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      const latestRelease = await consumer.getLatestRelease(unconstrainedVersionRange, true);
+      expect(latestRelease.id).to.equal(5);
+    });
   });
 
   it("gets correct assets for a release", async () => {
@@ -141,7 +163,7 @@ describe("Releases API consumer", () => {
 
     const consumer = new MockReleasesApiConsumer(owner, repo);
 
-    const assets = (await consumer.getLatestRelease(unconstrainedVersionConstraint)).assets;
+    const assets = (await consumer.getLatestRelease(unconstrainedVersionRange)).assets;
 
     expect(assets.length).to.equal(expectedAssets.length);
     expectedAssets.map((expectedAsset, index) => {
@@ -152,41 +174,6 @@ describe("Releases API consumer", () => {
   });
 });
 
-describe("Release version ordering", () => {
-  function createVersion(majorVersion: number, minorVersion: number, patchVersion: number, prereleaseVersion?: string, buildMetadata?: string): Version {
-    return {
-      buildMetadata,
-      majorVersion,
-      minorVersion,
-      patchVersion,
-      prereleaseVersion,
-      rawString: `${majorVersion}.${minorVersion}.${patchVersion}` +
-        prereleaseVersion ? `-${prereleaseVersion}` : "" +
-          buildMetadata ? `+${buildMetadata}` : ""
-    };
-  }
-
-  it("major versions compare correctly", () => {
-    expect(versionCompare(createVersion(3, 0, 0), createVersion(2, 9, 9)) > 0).to.be.true;
-  });
-
-  it("minor versions compare correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 0), createVersion(2, 0, 9)) > 0).to.be.true;
-  });
-
-  it("patch versions compare correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 2), createVersion(2, 1, 1)) > 0).to.be.true;
-  });
-
-  it("prerelease versions compare correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 0, "alpha.2"), createVersion(2, 1, 0, "alpha.1")) > 0).to.true;
-  });
-
-  it("build metadata compares correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 0, "alpha.1", "abcdef0"), createVersion(2, 1, 0, "alpha.1", "bcdef01"))).to.equal(0);
-  });
-});
-
 describe('Launcher path', () => {
   const pathToCmd = `abc${path.sep}codeql.cmd`;
   const pathToExe = `abc${path.sep}codeql.exe`;

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -1,4 +1,5 @@
 import * as path from 'path';
+import * as os from 'os';
 import { runTests } from 'vscode-test';
 
 // A subset of the fields in TestOptions from vscode-test, which we
@@ -11,25 +12,29 @@ type Suite = {
 };
 
 /**
- * Run an integration test suite `suite` at most `tries` times, or
- * until it succeeds, whichever comes first.
- *
- * TODO: Presently there is no way to distinguish a legitimately
- * failed test run from the test runner being terminated by a signal.
- * If in the future there arises a way to distinguish these cases
- * (e.g. https://github.com/microsoft/vscode-test/pull/56) only retry
- * in the terminated-by-signal case.
+ * Run an integration test suite `suite`, retrying if it segfaults, at
+ * most `tries` times.
  */
-async function runTestsWithRetry(suite: Suite, tries: number): Promise<void> {
+async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise<void> {
   for (let t = 0; t < tries; t++) {
     try {
       // Download and unzip VS Code if necessary, and run the integration test suite.
       await runTests(suite);
       return;
     } catch (err) {
-      console.error(`Exception raised while running tests: ${err}`);
-      if (t < tries - 1)
-        console.log('Retrying...');
+      if (err === 'SIGSEGV') {
+        console.error('Test runner segfaulted.');
+        if (t < tries - 1)
+          console.error('Retrying...');
+      }
+      else if (os.platform() === 'win32') {
+        console.error(`Test runner caught exception (${err})`);
+        if (t < tries - 1)
+          console.error('Retrying...');
+      }
+      else {
+        throw err;
+      }
     }
   }
   console.error(`Tried running suite ${tries} time(s), still failed, giving up.`);
@@ -67,7 +72,7 @@ async function main() {
     ];
 
     for (const integrationTestSuite of integrationTestSuites) {
-      await runTestsWithRetry(integrationTestSuite, 3);
+      await runTestsWithRetryOnSegfault(integrationTestSuite, 3);
     }
   } catch (err) {
     console.error(`Unexpected exception while running tests: ${err}`);

extensions/ql-vscode/test/data/library.qll

@@ -1,3 +1,3 @@
 predicate foo() {
- 1 == 1
-}
+ 1 = 1
+}

extensions/ql-vscode/test/pure-tests/command-lint.test.ts

@@ -0,0 +1,101 @@
+import { expect } from 'chai';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+
+type CmdDecl = {
+  command: string;
+  when?: string;
+  title?: string;
+}
+
+describe('commands declared in package.json', function() {
+  const manifest = fs.readJsonSync(path.join(__dirname, '../../package.json'));
+  const commands = manifest.contributes.commands;
+  const menus = manifest.contributes.menus;
+
+  const disabledInPalette: Set<string> = new Set<string>();
+
+  // These commands should appear in the command palette, and so
+  // should be prefixed with 'CodeQL: '.
+  const paletteCmds: Set<string> = new Set<string>();
+
+  // These commands arising on context menus in non-CodeQL controlled
+  // panels, (e.g. file browser) and so should be prefixed with 'CodeQL: '.
+  const contribContextMenuCmds: Set<string> = new Set<string>();
+
+  // These are commands used in CodeQL controlled panels, and so don't need any prefixing in their title.
+  const scopedCmds: Set<string> = new Set<string>();
+  const commandTitles: { [cmd: string]: string } = {};
+
+  commands.forEach((commandDecl: CmdDecl) => {
+    const { command, title } = commandDecl;
+    if (command.match(/^codeQL\./)
+      || command.match(/^codeQLQueryResults\./)
+      || command.match(/^codeQLTests\./)) {
+      paletteCmds.add(command);
+      expect(title).not.to.be.undefined;
+      commandTitles[command] = title!;
+    }
+    else if (command.match(/^codeQLDatabases\./)
+      || command.match(/^codeQLQueryHistory\./)) {
+      scopedCmds.add(command);
+      expect(title).not.to.be.undefined;
+      commandTitles[command] = title!;
+    }
+    else {
+      expect.fail(`Unexpected command name ${command}`);
+    }
+  });
+
+  menus['explorer/context'].forEach((commandDecl: CmdDecl) => {
+    const { command } = commandDecl;
+    paletteCmds.delete(command);
+    contribContextMenuCmds.add(command);
+  });
+
+  menus['editor/context'].forEach((commandDecl: CmdDecl) => {
+    const { command } = commandDecl;
+    paletteCmds.delete(command);
+    contribContextMenuCmds.add(command);
+  });
+
+  menus.commandPalette.forEach((commandDecl: CmdDecl) => {
+    if (commandDecl.when === 'false')
+      disabledInPalette.add(commandDecl.command);
+  });
+
+
+
+  it('should have commands appropriately prefixed', function() {
+    paletteCmds.forEach(command => {
+      expect(commandTitles[command], `command ${command} should be prefixed with 'CodeQL: ', since it is accessible from the command palette`).to.match(/^CodeQL: /);
+    });
+
+    contribContextMenuCmds.forEach(command => {
+      expect(commandTitles[command], `command ${command} should be prefixed with 'CodeQL: ', since it is accessible from a context menu in a non-extension-controlled context`).to.match(/^CodeQL: /);
+    });
+
+    scopedCmds.forEach(command => {
+      expect(commandTitles[command], `command ${command} should not be prefixed with 'CodeQL: ', since it is accessible from an extension-controlled context`).not.to.match(/^CodeQL: /);
+    });
+  });
+
+  it('should have the right commands accessible from the command palette', function() {
+    paletteCmds.forEach(command => {
+      expect(disabledInPalette.has(command), `command ${command} should be enabled in the command palette`).to.be.false;
+    });
+
+    // Commands in contribContextMenuCmds may reasonbly be enabled or
+    // disabled in the command palette; for example, codeQL.runQuery
+    // is available there, since we heuristically figure out which
+    // query to run, but codeQL.setCurrentDatabase is not.
+
+    scopedCmds.forEach(command => {
+      expect(disabledInPalette.has(command), `command ${command} should be disabled in the command palette`).to.be.true;
+    });
+  });
+
+
+});
+
+

rush.json

@@ -9,7 +9,8 @@
   "pnpmOptions": {
     "strictPeerDependencies": true
   },
-  "nodeSupportedVersionRange": ">=10.13.0 <13.0.0",
+  "nodeSupportedVersionRange": ">=10.13.0 <15.0.0",
+  "suppressNodeLtsWarning": true,
   "ensureConsistentVersions": true,
   "projectFolderMinDepth": 2,
   "projectFolderMaxDepth": 2,