Add telemetry for commands

This commit adds telemetry capturing for command execution. The data captured explicitly captured and sent to application insights is only the command id, execution time, and command completion status. We also capture errors thrown by any command execution, but these are not sent to application insights. Telemetry capturing is opt-in. No data will be sent to application insights unless the user explicitly allows it. There are two new config settings added. The first controls whether or not telemetry should be sent. This setting AND the global telemetry setting must be enabled in order for telemetry to be sent. The second setting controls whether or not telemetry event data should be logged to the extension console. The hope here is that users can inspect exactly what data is sent to the server and can have confidence that nothing concerning is being leaked. Note that the global setting for disabling telemetry collection is handled inside the `vscode-extension-telemetry` package implicitly, so this extension doesn't touch that setting explicitly. The `codeql.canary` setting is being used to add an additional flag to telemetry events. This flag will help us determine if a user in internal or not. The application insights key is injected at build time through a repository secret. This commit also includes a new `TELEMETRY.md` file that explains what is being captured, and why.
2020-10-06 11:17:57 -07:00
parent f154206b47
commit 292e695646
23 changed files with 991 additions and 40 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -30,6 +30,8 @@ jobs:

      - name: Build
        working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
        run: |
          npm run build
        shell: bash
@@ -71,6 +73,8 @@ jobs:

      - name: Build
        working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
        run: |
          npm run build
        shell: bash
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -20,7 +20,6 @@ jobs:
  build:
    name: Release
    runs-on: ubuntu-latest
-    # TODO Share steps with the main workflow.
    steps:
      - name: Checkout
        uses: actions/checkout@v2
@@ -36,7 +35,10 @@ jobs:
        shell: bash

      - name: Build
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
        run: |
+          echo "APP INSIGHTS KEY LENGTH: ${#APP_INSIGHTS_KEY}"
          cd extensions/ql-vscode
          npm run build -- --release
        shell: bash
@@ -65,6 +67,7 @@ jobs:

      - name: Create release
        id: create-release
+        if: github.event_name == 'pull_request'
        uses: actions/create-release@v1.0.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -79,7 +82,7 @@ jobs:

      - name: Upload release asset
        uses: actions/upload-release-asset@v1.0.1
-        if: success()
+        if: success() && github.event_name == 'pull_request'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
@@ -90,16 +93,27 @@ jobs:
          asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
          asset_content_type: application/zip

+
+      - name: No Release
+        if: github.event_name != 'pull_request'
+        run: |
+          echo "Not making a release because this is not a pull request"
+
+      ###
+      # Do Post release work: version bump and changelog PR
+      # Only do this if we are running from a PR (ie- this is part of the release process)
+
      # The checkout action does not fetch the main branch.
      # Fetch the main branch so that we can base the version bump PR against main.
      - name: Fetch main branch
+        if: github.event_name == 'pull_request'
        run: |
          git fetch --depth=1 origin main:main
          git checkout main

      - name: Bump patch version
        id: bump-patch-version
-        if: success()
+        if: success() && github.event_name == 'pull_request'
        run: |
          cd extensions/ql-vscode
          # Bump to the next patch version. Major or minor version bumps will have to be done manually.
@@ -108,14 +122,14 @@ jobs:
          echo "::set-output name=next_version::$NEXT_VERSION"

      - name: Add changelog for next release
-        if: success()
+        if: success() && github.event_name == 'pull_request'
        run: |
          cd extensions/ql-vscode
          perl -i -pe 's/^/## \[UNRELEASED\]\n\n/ if($.==3)' CHANGELOG.md

      - name: Create version bump PR
        uses: peter-evans/create-pull-request@c7f493a8000b8aeb17a1332e326ba76b57cb83eb # v3.4.1
-        if: success()
+        if: success() && github.event_name == 'pull_request'
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          commit-message: Bump version to ${{ steps.bump-patch-version.outputs.next_version }}
--- a/extensions/ql-vscode/CHANGELOG.md
+++ b/extensions/ql-vscode/CHANGELOG.md
@@ -4,6 +4,7 @@

 - Fix bug where databases are not reregistered when the query server restarts. [#734](https://github.com/github/vscode-codeql/pull/734)
 - Fix bug where upgrade requests were erroneously being marked as failed. [#734](https://github.com/github/vscode-codeql/pull/734)
+- Capture usage data from users. See [TELEMETRY.md](https://github.com/github/vscode-codeql/blob/main/TELEMETRY.md) for more information. [#611](https://github.com/github/vscode-codeql/pull/611)

 ## 1.3.10 - 20 January 2021

--- a/extensions/ql-vscode/README.md
+++ b/extensions/ql-vscode/README.md
@@ -110,3 +110,7 @@ For more information about the CodeQL extension, [see the documentation](https:/
 ## License

 The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the MIT License. The version of CodeQL used by the CodeQL extension is subject to the [GitHub CodeQL Terms & Conditions](https://securitylab.github.com/tools/codeql/license).
+
+## Data and Telemetry
+
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. Please see [telemetry](TELEMETRY.md) for more information.
--- a/extensions/ql-vscode/TELEMETRY.md
+++ b/extensions/ql-vscode/TELEMETRY.md
@@ -0,0 +1,45 @@
+# Telemetry in the CodeQL extension for VS Code
+
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days.
+
+## Why do you collect data?
+
+GitHub collects aggregated, anonymous usage data and metrics to help us improve CodeQL for VS Code. IP addresses and installation IDs are collected only to ensure that anonymous data is not duplicated during aggregation.
+
+## What data is collected
+
+GitHub collects the following information related to the usage of the extension. The data collected are:
+
+- The identifiers of any CodeQL-related [VS Code commands](https://code.visualstudio.com/docs/getstarted/tips-and-tricks#_command-palette) that are run
+- For each command: the timestamp, time taken, and whether or not the command completed successfully
+- VS Code and extension version
+- Randomly generated GUID that uniquely identifies a CodeQL extension installation. (Discarded before aggregation.)
+- IP address of the client sending the telemetry data. (Discarded before aggregation.)
+- Whether or not the `codeQL.canary` setting is enabled and set to `true`
+
+## How long will data be retained?
+
+IP address and GUIDs will be retained for a maximum of 30 days. Anonymous, aggregated data that includes command identifiers, run times, and timestamps will be retained for a maximum of 180 days.
+
+## Who will have access to this data?
+
+IP address and GUIDs will only be available to the core developers of CodeQL. Aggregated data will be available to GitHub employees.
+
+## What data is **NOT** collected?
+
+We only collect the minimal amount of data we need to answer the questions about how our users are experiencing this product. To that end, we do not collect the following information:
+
+- No GitHub user ID
+- No CodeQL database names or contents
+- No contents of CodeQL queries
+- No filesystem paths.
+
+## How do I disable telemetry reporting?
+
+You can disable telemetry collection by setting `codeQL.telemetry.enableTelemetry` to `false` in [your settings](https://code.visualstudio.com/docs/getstarted/settings#_settings-editor). Telemetry collection is disabled by default.
+
+Additionally, telemetry collection will be disabled if the global `telemetry.enableTelemetry` setting is set to `false`. For more information on global telemetry collection, see [Microsoft’s documentation](https://code.visualstudio.com/docs/supporting/faq#_how-to-disable-telemetry-reporting).
+
+## More information
+
+See GitHub's [Privacy Statement](https://docs.github.com/en/free-pro-team@latest/github/site-policy/github-privacy-statement) and [Terms of Service](https://docs.github.com/en/free-pro-team@latest/github/site-policy/github-terms-of-service) for more information.
--- a/extensions/ql-vscode/gulpfile.ts/appInsights.ts
+++ b/extensions/ql-vscode/gulpfile.ts/appInsights.ts
@@ -0,0 +1,16 @@
+import * as gulp from 'gulp';
+import * as replace from 'gulp-replace';
+
+/** Inject the application insights key into the telemetry file */
+export function injectAppInsightsKey() {
+  if (!process.env.APP_INSIGHTS_KEY) {
+    // noop
+    console.log('APP_INSIGHTS_KEY environment variable is not set. So, cannot inject it into the application.');
+    return Promise.resolve();
+  }
+
+  // replace the key
+  return gulp.src(['out/telemetry.js'])
+    .pipe(replace(/REPLACE-APP-INSIGHTS-KEY/, process.env.APP_INSIGHTS_KEY))
+    .pipe(gulp.dest('out/'));
+}
--- a/extensions/ql-vscode/gulpfile.ts/index.ts
+++ b/extensions/ql-vscode/gulpfile.ts/index.ts
@@ -4,7 +4,12 @@ import { compileTextMateGrammar } from './textmate';
 import { copyTestData } from './tests';
 import { compileView } from './webpack';
 import { packageExtension } from './package';
+import { injectAppInsightsKey } from './appInsights';

-export const buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
-export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData };
-exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);
+export const buildWithoutPackage =
+  gulp.parallel(
+    compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
+  );
+
+export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData, injectAppInsightsKey };
+export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);
--- a/extensions/ql-vscode/package-lock.json
+++ b/extensions/ql-vscode/package-lock.json
@@ -268,6 +268,15 @@
        "chokidar": "^2.1.2"
      }
    },
+    "@types/gulp-replace": {
+      "version": "0.0.31",
+      "resolved": "https://registry.npmjs.org/@types/gulp-replace/-/gulp-replace-0.0.31.tgz",
+      "integrity": "sha512-dbgQ1u0N9ShXrzahBgQfMSu6qUh8nlTLt7whhQ0S0sEUHhV3scysppJ1UX0fl53PJENgAL99ueykddyrCaDt7g==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/gulp-sourcemaps": {
      "version": "0.0.32",
      "resolved": "https://registry.npmjs.org/@types/gulp-sourcemaps/-/gulp-sourcemaps-0.0.32.tgz",
@@ -952,6 +961,18 @@
        "buffer-equal": "^1.0.0"
      }
    },
+    "applicationinsights": {
+      "version": "1.8.7",
+      "resolved": "https://registry.npmjs.org/applicationinsights/-/applicationinsights-1.8.7.tgz",
+      "integrity": "sha512-+HENzPBdSjnWL9mc+9o+j9pEaVNI4WsH5RNvfmRLfwQYvbJumcBi4S5bUzclug5KCcFP0S4bYJOmm9MV3kv2GA==",
+      "dev": true,
+      "requires": {
+        "cls-hooked": "^4.2.2",
+        "continuation-local-storage": "^3.2.1",
+        "diagnostic-channel": "0.3.1",
+        "diagnostic-channel-publishers": "0.4.1"
+      }
+    },
    "aproba": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -1230,6 +1251,30 @@
      "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==",
      "dev": true
    },
+    "async-hook-jl": {
+      "version": "1.7.6",
+      "resolved": "https://registry.npmjs.org/async-hook-jl/-/async-hook-jl-1.7.6.tgz",
+      "integrity": "sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==",
+      "requires": {
+        "stack-chain": "^1.3.7"
+      }
+    },
+    "async-listener": {
+      "version": "0.6.10",
+      "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz",
+      "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==",
+      "requires": {
+        "semver": "^5.3.0",
+        "shimmer": "^1.1.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
    "async-settle": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz",
@@ -1376,6 +1421,12 @@
      "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==",
      "dev": true
    },
+    "binaryextensions": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-2.3.0.tgz",
+      "integrity": "sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg==",
+      "dev": true
+    },
    "bindings": {
      "version": "1.5.0",
      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
@@ -2024,6 +2075,23 @@
        "readable-stream": "^2.3.5"
      }
    },
+    "cls-hooked": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/cls-hooked/-/cls-hooked-4.2.2.tgz",
+      "integrity": "sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==",
+      "requires": {
+        "async-hook-jl": "^1.7.6",
+        "emitter-listener": "^1.0.1",
+        "semver": "^5.4.1"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
    "code-point-at": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
@@ -2151,6 +2219,15 @@
      "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=",
      "dev": true
    },
+    "continuation-local-storage": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz",
+      "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==",
+      "requires": {
+        "async-listener": "^0.6.0",
+        "emitter-listener": "^1.1.1"
+      }
+    },
    "convert-source-map": {
      "version": "1.7.0",
      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz",
@@ -2599,6 +2676,29 @@
      "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=",
      "dev": true
    },
+    "diagnostic-channel": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/diagnostic-channel/-/diagnostic-channel-0.3.1.tgz",
+      "integrity": "sha512-6eb9YRrimz8oTr5+JDzGmSYnXy5V7YnK5y/hd8AUDK1MssHjQKm9LlD6NSrHx4vMDF3+e/spI2hmWTviElgWZA==",
+      "dev": true,
+      "requires": {
+        "semver": "^5.3.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "dev": true
+        }
+      }
+    },
+    "diagnostic-channel-publishers": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/diagnostic-channel-publishers/-/diagnostic-channel-publishers-0.4.1.tgz",
+      "integrity": "sha512-NpZ7IOVUfea/kAx4+ub4NIYZyRCSymjXM5BZxnThs3ul9gAKqjm7J8QDDQW3Ecuo2XxjNLoWLeKmrPUWKNZaYw==",
+      "dev": true
+    },
    "diff": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -2704,6 +2804,12 @@
        "object.defaults": "^1.1.0"
      }
    },
+    "editions": {
+      "version": "1.3.4",
+      "resolved": "https://registry.npmjs.org/editions/-/editions-1.3.4.tgz",
+      "integrity": "sha512-gzao+mxnYDzIysXKMQi/+M1mjy/rjestjg6OPoYTtI+3Izp23oiGZitsl9lPDPiTGXbcSIk1iJWhliSaglxnUg==",
+      "dev": true
+    },
    "editorconfig": {
      "version": "0.15.3",
      "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz",
@@ -2753,6 +2859,14 @@
        }
      }
    },
+    "emitter-listener": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz",
+      "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==",
+      "requires": {
+        "shimmer": "^1.2.0"
+      }
+    },
    "emoji-regex": {
      "version": "8.0.0",
      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -4011,6 +4125,17 @@
        }
      }
    },
+    "gulp-replace": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/gulp-replace/-/gulp-replace-1.0.0.tgz",
+      "integrity": "sha512-lgdmrFSI1SdhNMXZQbrC75MOl1UjYWlOWNbNRnz+F/KHmgxt3l6XstBoAYIdadwETFyG/6i+vWUSCawdC3pqOw==",
+      "dev": true,
+      "requires": {
+        "istextorbinary": "2.2.1",
+        "readable-stream": "^2.0.1",
+        "replacestream": "^4.0.0"
+      }
+    },
    "gulp-sourcemaps": {
      "version": "2.6.5",
      "resolved": "https://registry.npmjs.org/gulp-sourcemaps/-/gulp-sourcemaps-2.6.5.tgz",
@@ -4904,6 +5029,17 @@
      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
      "dev": true
    },
+    "istextorbinary": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-2.2.1.tgz",
+      "integrity": "sha512-TS+hoFl8Z5FAFMK38nhBkdLt44CclNRgDHWeMgsV8ko3nDlr/9UI2Sf839sW7enijf8oKsZYXRvM8g0it9Zmcw==",
+      "dev": true,
+      "requires": {
+        "binaryextensions": "2",
+        "editions": "^1.3.3",
+        "textextensions": "2"
+      }
+    },
    "js-tokens": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -7547,6 +7683,17 @@
        "remove-trailing-separator": "^1.1.0"
      }
    },
+    "replacestream": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/replacestream/-/replacestream-4.0.3.tgz",
+      "integrity": "sha512-AC0FiLS352pBBiZhd4VXB1Ab/lh0lEgpP+GGvZqbQh8a5cmXVoTe5EX/YeTFArnp4SRGTHh1qCHu9lGs1qG8sA==",
+      "dev": true,
+      "requires": {
+        "escape-string-regexp": "^1.0.3",
+        "object-assign": "^4.0.1",
+        "readable-stream": "^2.0.2"
+      }
+    },
    "require-directory": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -7814,6 +7961,11 @@
      "integrity": "sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==",
      "dev": true
    },
+    "shimmer": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz",
+      "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw=="
+    },
    "side-channel": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.2.tgz",
@@ -8122,6 +8274,11 @@
        "figgy-pudding": "^3.5.1"
      }
    },
+    "stack-chain": {
+      "version": "1.3.7",
+      "resolved": "https://registry.npmjs.org/stack-chain/-/stack-chain-1.3.7.tgz",
+      "integrity": "sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU="
+    },
    "stack-trace": {
      "version": "0.0.10",
      "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
@@ -8505,6 +8662,12 @@
      "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
      "dev": true
    },
+    "textextensions": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-2.6.0.tgz",
+      "integrity": "sha512-49WtAWS+tcsy93dRt6P0P3AMD2m5PvXRhuEA0kaXos5ZLlujtYmpmFsB+QvWUSxE1ZsstmYXfQ7L40+EcQgpAQ==",
+      "dev": true
+    },
    "through": {
      "version": "2.3.8",
      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
@@ -9211,6 +9374,45 @@
        }
      }
    },
+    "vscode-extension-telemetry": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.1.6.tgz",
+      "integrity": "sha512-rbzSg7k4NnsCdF4Lz0gI4jl3JLXR0hnlmfFgsY8CSDYhXgdoIxcre8jw5rjkobY0xhSDhbG7xCjP8zxskySJ/g==",
+      "requires": {
+        "applicationinsights": "1.7.4"
+      },
+      "dependencies": {
+        "applicationinsights": {
+          "version": "1.7.4",
+          "resolved": "https://registry.npmjs.org/applicationinsights/-/applicationinsights-1.7.4.tgz",
+          "integrity": "sha512-XFLsNlcanpjFhHNvVWEfcm6hr7lu9znnb6Le1Lk5RE03YUV9X2B2n2MfM4kJZRrUdV+C0hdHxvWyv+vWoLfY7A==",
+          "requires": {
+            "cls-hooked": "^4.2.2",
+            "continuation-local-storage": "^3.2.1",
+            "diagnostic-channel": "0.2.0",
+            "diagnostic-channel-publishers": "^0.3.3"
+          }
+        },
+        "diagnostic-channel": {
+          "version": "0.2.0",
+          "resolved": "https://registry.npmjs.org/diagnostic-channel/-/diagnostic-channel-0.2.0.tgz",
+          "integrity": "sha1-zJmvlhLCP7H/8TYSxy8sv6qNWhc=",
+          "requires": {
+            "semver": "^5.3.0"
+          }
+        },
+        "diagnostic-channel-publishers": {
+          "version": "0.3.5",
+          "resolved": "https://registry.npmjs.org/diagnostic-channel-publishers/-/diagnostic-channel-publishers-0.3.5.tgz",
+          "integrity": "sha512-AOIjw4T7Nxl0G2BoBPhkQ6i7T4bUd9+xvdYizwvG7vVAM1dvr+SDrcUudlmzwH0kbEwdR2V1EcnKT0wAeYLQNQ=="
+        },
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
    "vscode-jsonrpc": {
      "version": "5.0.1",
      "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz",
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@@ -174,6 +174,19 @@
          "minimum": 0,
          "maximum": 1024,
          "description": "Number of threads for running CodeQL tests."
+        },
+        "codeQL.telemetry.enableTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "markdownDescription": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub. For more information, see [TELEMETRY.md](https://github.com/github/vscode-codeql/blob/main/TELEMETRY.md)",
+          "description": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub."
+        },
+        "codeQL.telemetry.logTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "description": "Specifies whether or not to write telemetry events to the extension log."
        }
      }
    },
@@ -735,6 +748,7 @@
    "tmp-promise": "~3.0.2",
    "tree-kill": "~1.2.2",
    "unzipper": "~0.10.5",
+    "vscode-extension-telemetry": "^0.1.6",
    "vscode-jsonrpc": "^5.0.1",
    "vscode-languageclient": "^6.1.3",
    "vscode-test-adapter-api": "~1.7.0",
@@ -750,6 +764,7 @@
    "@types/glob": "^7.1.1",
    "@types/google-protobuf": "^3.2.7",
    "@types/gulp": "^4.0.6",
+    "@types/gulp-replace": "0.0.31",
    "@types/gulp-sourcemaps": "0.0.32",
    "@types/js-yaml": "^3.12.5",
    "@types/jszip": "~3.1.6",
@@ -772,6 +787,7 @@
    "@typescript-eslint/eslint-plugin": "~2.23.0",
    "@typescript-eslint/parser": "~2.23.0",
    "ansi-colors": "^4.1.1",
+    "applicationinsights": "^1.8.7",
    "chai": "^4.2.0",
    "chai-as-promised": "~7.1.1",
    "css-loader": "~3.1.0",
@@ -779,6 +795,7 @@
    "eslint-plugin-react": "~7.19.0",
    "glob": "^7.1.4",
    "gulp": "^4.0.2",
+    "gulp-replace": "^1.0.0",
    "gulp-sourcemaps": "^2.6.5",
    "gulp-typescript": "^5.0.1",
    "husky": "~4.2.5",
--- a/extensions/ql-vscode/src/commandRunner.ts
+++ b/extensions/ql-vscode/src/commandRunner.ts
@@ -8,6 +8,7 @@ import {
 } from 'vscode';
 import { showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
 import { logger } from './logging';
+import { telemetryListener } from './telemetry';

 export class UserCancellationException extends Error {
  /**
@@ -114,9 +115,13 @@ export function commandRunner(
  task: NoProgressTask,
 ): Disposable {
  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+
    try {
      return await task(...args);
    } catch (e) {
+      error = e;
      const errorMessage = `${e.message || e} (${commandId})`;
      if (e instanceof UserCancellationException) {
        // User has cancelled this action manually
@@ -135,6 +140,9 @@ export function commandRunner(
        });
      }
      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
    }
  });
 }
@@ -155,6 +163,8 @@ export function commandRunnerWithProgress<R>(
  progressOptions: Partial<ProgressOptions>
 ): Disposable {
  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
    const progressOptionsWithDefaults = {
      location: ProgressLocation.Notification,
      ...progressOptions
@@ -162,6 +172,7 @@ export function commandRunnerWithProgress<R>(
    try {
      return await withProgress(progressOptionsWithDefaults, task, ...args);
    } catch (e) {
+      error = e;
      const errorMessage = `${e.message || e} (${commandId})`;
      if (e instanceof UserCancellationException) {
        // User has cancelled this action manually
@@ -180,6 +191,9 @@ export function commandRunnerWithProgress<R>(
        });
      }
      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
    }
  });
 }
--- a/extensions/ql-vscode/src/config.ts
+++ b/extensions/ql-vscode/src/config.ts
@@ -4,7 +4,7 @@ import { DistributionManager } from './distribution';
 import { logger } from './logging';

 /** Helper class to look up a labelled (and possibly nested) setting. */
-class Setting {
+export class Setting {
  name: string;
  parent?: Setting;

@@ -39,8 +39,16 @@ class Setting {

 const ROOT_SETTING = new Setting('codeQL');

-// Distribution configuration
+// Global configuration
+const TELEMETRY_SETTING = new Setting('telemetry', ROOT_SETTING);
+const GLOBAL_TELEMETRY_SETTING = new Setting('telemetry');

+export const LOG_TELEMETRY = new Setting('logTelemetry', TELEMETRY_SETTING);
+export const ENABLE_TELEMETRY = new Setting('enableTelemetry', TELEMETRY_SETTING);
+
+export const GLOBAL_ENABLE_TELEMETRY = new Setting('enableTelemetry', GLOBAL_TELEMETRY_SETTING);
+
+// Distribution configuration
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
 const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
@@ -104,7 +112,7 @@ export interface CliConfig {
 }


-abstract class ConfigListener extends DisposableObject {
+export abstract class ConfigListener extends DisposableObject {
  protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());

  constructor() {
@@ -244,7 +252,7 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
 /**
 * Enables canary features of this extension. Recommended for all internal users.
 */
-const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);
+export const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);

 export function isCanary() {
  return !!CANARY_FEATURES.getValue<boolean>();
--- a/extensions/ql-vscode/src/databases-ui.ts
+++ b/extensions/ql-vscode/src/databases-ui.ts
@@ -23,9 +23,9 @@ import {
  ProgressCallback,
 } from './commandRunner';
 import {
-  showAndLogErrorMessage,
  isLikelyDatabaseRoot,
-  isLikelyDbLanguageFolder
+  isLikelyDbLanguageFolder,
+  showAndLogErrorMessage
 } from './helpers';
 import { logger } from './logging';
 import { clearCacheInDatabase } from './run-queries';
--- a/extensions/ql-vscode/src/databases.ts
+++ b/extensions/ql-vscode/src/databases.ts
@@ -8,7 +8,7 @@ import {
  showAndLogErrorMessage,
  showAndLogWarningMessage,
  showAndLogInformationMessage,
-  isLikelyDatabaseRoot,
+  isLikelyDatabaseRoot
 } from './helpers';
 import {
  ProgressCallback,
--- a/extensions/ql-vscode/src/distribution.ts
+++ b/extensions/ql-vscode/src/distribution.ts
@@ -225,9 +225,11 @@ export class DistributionManager implements DistributionProvider {
   *
   * Returns a failed promise if an unexpected error occurs during installation.
   */
-  public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: ProgressCallback): Promise<void> {
-    return this.extensionSpecificDistributionManager!.installDistributionRelease(release, progressCallback);
+  public installExtensionManagedDistributionRelease(
+    release: Release,
+    progressCallback?: ProgressCallback
+  ): Promise<void> {
+    return this.extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
  }

  public get onDidChangeDistribution(): Event<void> | undefined {
--- a/extensions/ql-vscode/src/extension.ts
+++ b/extensions/ql-vscode/src/extension.ts
@@ -45,7 +45,6 @@ import {
  GithubRateLimitedError
 } from './distribution';
 import * as helpers from './helpers';
-import { commandRunner, commandRunnerWithProgress, ProgressCallback, ProgressUpdate, withProgress } from './commandRunner';
 import { assertNever } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
 import { InterfaceManager } from './interface';
@@ -60,6 +59,14 @@ import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
 import { CompareInterfaceManager } from './compare/compare-interface';
 import { gatherQlFiles } from './pure/files';
+import { initializeTelemetry } from './telemetry';
+import {
+  commandRunner,
+  commandRunnerWithProgress,
+  ProgressCallback,
+  withProgress,
+  ProgressUpdate
+} from './commandRunner';

 /**
 * extension.ts
@@ -88,6 +95,9 @@ const errorStubs: Disposable[] = [];
 */
 let isInstallingOrUpdatingDistribution = false;

+const extensionId = 'GitHub.vscode-codeql';
+const extension = extensions.getExtension(extensionId);
+
 /**
 * If the user tries to execute vscode commands after extension activation is failed, give
 * a sensible error message.
@@ -98,8 +108,6 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
  // Remove existing stubs
  errorStubs.forEach(stub => stub.dispose());

-  const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
-  const extension = extensions.getExtension(extensionId);
  if (extension === undefined) {
    throw new Error(`Can't find extension ${extensionId}`);
  }
@@ -139,10 +147,14 @@ export interface CodeQLExtensionInterface {
 * @returns CodeQLExtensionInterface
 */
 export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | {}> {
-  logger.log('Starting CodeQL extension');
+  logger.log(`Starting ${extensionId} extension`);
+  if (extension === undefined) {
+    throw new Error(`Can't find extension ${extensionId}`);
+  }

  const distributionConfigListener = new DistributionConfigListener();
  initializeLogging(ctx);
+  await initializeTelemetry(extension, ctx);
  languageSupport.install();

  ctx.subscriptions.push(distributionConfigListener);
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@@ -80,14 +80,23 @@ async function internalShowAndLog(

 /**
 * Opens a modal dialog for the user to make a yes/no choice.
- * @param message The message to show.
 *
- * @return `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
 */
-export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
+export async function showBinaryChoiceDialog(message: string, modal = true): Promise<boolean | undefined> {
  const yesItem = { title: 'Yes', isCloseAffordance: false };
  const noItem = { title: 'No', isCloseAffordance: true };
-  const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
+  const chosenItem = await Window.showInformationMessage(message, { modal }, yesItem, noItem);
+  if (!chosenItem) {
+    return undefined;
+  }
  return chosenItem?.title === yesItem.title;
 }

--- a/extensions/ql-vscode/src/interface.ts
+++ b/extensions/ql-vscode/src/interface.ts
@@ -30,8 +30,8 @@ import {
  RawResultsSortState,
 } from './pure/interface-types';
 import { Logger } from './logging';
-import { commandRunner } from './commandRunner';
 import * as messages from './pure/messages';
+import { commandRunner } from './commandRunner';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
--- a/extensions/ql-vscode/src/query-history.ts
+++ b/extensions/ql-vscode/src/query-history.ts
@@ -10,13 +10,11 @@ import {
  showAndLogWarningMessage,
  showBinaryChoiceDialog
 } from './helpers';
-import {
-  commandRunner
-} from './commandRunner';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
 import { DisposableObject } from './vscode-utils/disposable-object';
+import { commandRunner } from './commandRunner';

 /**
 * query-history.ts
--- a/extensions/ql-vscode/src/telemetry.ts
+++ b/extensions/ql-vscode/src/telemetry.ts
@@ -0,0 +1,220 @@
+import { ConfigurationTarget, Extension, ExtensionContext, ConfigurationChangeEvent } from 'vscode';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY } from './config';
+import * as appInsights from 'applicationinsights';
+import { logger } from './logging';
+import { UserCancellationException } from './commandRunner';
+import { showBinaryChoiceDialog } from './helpers';
+
+// Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
+const key = 'REPLACE-APP-INSIGHTS-KEY';
+
+export enum CommandCompletion {
+  Success = 'Success',
+  Failed = 'Failed',
+  Cancelled = 'Cancelled'
+}
+
+// Avoid sending the following data to App insights since we don't need it.
+const tagsToRemove = [
+  'ai.application.ver',
+  'ai.device.id',
+  'ai.cloud.roleInstance',
+  'ai.cloud.role',
+  'ai.device.id',
+  'ai.device.osArchitecture',
+  'ai.device.osPlatform',
+  'ai.device.osVersion',
+  'ai.internal.sdkVersion',
+  'ai.session.id'
+];
+
+const baseDataPropertiesToRemove = [
+  'common.os',
+  'common.platformversion',
+  'common.remotename',
+  'common.uikind',
+  'common.vscodesessionid'
+];
+
+export class TelemetryListener extends ConfigListener {
+
+  static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
+
+  private reporter?: TelemetryReporter;
+
+  constructor(
+    private readonly id: string,
+    private readonly version: string,
+    private readonly key: string,
+    private readonly ctx: ExtensionContext
+  ) {
+    super();
+  }
+
+  /**
+   * This function handles changes to relevant configuration elements. There are 2 configuration
+   * ids that this function cares about:
+   *
+   *     * `codeQL.telemetry.enableTelemetry`: If this one has changed, then we need to re-initialize
+   *        the reporter and the reporter may wind up being removed.
+   *     * `codeQL.canary`: A change here could possibly re-trigger a dialog popup.
+   *
+   * Note that the global telemetry setting also gate-keeps whether or not to send telemetry events
+   * to Application Insights. However, this gatekeeping happens inside of the vscode-extension-telemetry
+   * package. So, this does not need to be handled here.
+   *
+   * @param e the configuration change event
+   */
+  async handleDidChangeConfiguration(e: ConfigurationChangeEvent): Promise<void> {
+    if (e.affectsConfiguration('codeQL.telemetry.enableTelemetry')) {
+      await this.initialize();
+    }
+
+    // Re-request telemetry so that users can see the dialog again.
+    // Re-request if codeQL.canary is being set to `true` and telemetry
+    // is not currently enabled.
+    if (
+      e.affectsConfiguration('codeQL.canary') &&
+      CANARY_FEATURES.getValue() &&
+      !ENABLE_TELEMETRY.getValue()
+    ) {
+      await Promise.all([
+        this.setTelemetryRequested(false),
+        this.requestTelemetryPermission()
+      ]);
+    }
+  }
+
+  async initialize() {
+    await this.requestTelemetryPermission();
+
+    this.disposeReporter();
+
+    if (ENABLE_TELEMETRY.getValue<boolean>()) {
+      this.createReporter();
+    }
+  }
+
+  private createReporter() {
+    this.reporter = new TelemetryReporter(
+      this.id,
+      this.version,
+      this.key,
+      /* anonymize stack traces */ true
+    );
+    this.push(this.reporter);
+
+    const client = (this.reporter as any).appInsightsClient as appInsights.TelemetryClient;
+    if (client) {
+      // add a telemetry processor to delete unwanted properties
+      client.addTelemetryProcessor((envelope: any) => {
+        tagsToRemove.forEach(tag => delete envelope.tags[tag]);
+        const baseDataProperties = (envelope.data as any)?.baseData?.properties;
+        if (baseDataProperties) {
+          baseDataPropertiesToRemove.forEach(prop => delete baseDataProperties[prop]);
+        }
+
+        return true;
+      });
+
+      // add a telemetry processor to log if requested
+      client.addTelemetryProcessor((envelope) => {
+        if (LOG_TELEMETRY.getValue<boolean>()) {
+          logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
+        }
+        return true;
+      });
+    }
+  }
+
+  dispose() {
+    super.dispose();
+    this.reporter?.dispose();
+  }
+
+  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+    if (!this.reporter) {
+      return;
+    }
+    const status = !error
+      ? CommandCompletion.Success
+      : error instanceof UserCancellationException
+        ? CommandCompletion.Cancelled
+        : CommandCompletion.Failed;
+
+    const isCanary = (!!CANARY_FEATURES.getValue<boolean>()).toString();
+
+    this.reporter.sendTelemetryEvent(
+      'command-usage',
+      {
+        name,
+        status,
+        isCanary
+      },
+      { executionTime }
+    );
+  }
+
+  /**
+   * Displays a popup asking the user if they want to enable telemetry
+   * for this extension.
+   */
+  async requestTelemetryPermission() {
+    if (!this.wasTelemetryRequested()) {
+      // if global telemetry is disabled, avoid showing the dialog or making any changes
+      let result = undefined;
+      if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
+        // Extension won't start until this completes.
+        result = await showBinaryChoiceDialog(
+          'Do we have your permission to collect usage data and metrics to help us improve CodeQL for VSCode? See [TELEMETRY.md](https://github.com/github/vscode-codeql/blob/main/TELEMETRY.md) for details of what we collect and how we use it.',
+          // We make this dialog modal for now.
+          // If we do decide to keep this dialog as modal, then this implementation can change and
+          // we no longer need to call Promise.race. Before committing this PR, we need to make
+          // this decision.
+          true
+        );
+      }
+      if (result !== undefined) {
+        await Promise.all([
+          this.setTelemetryRequested(true),
+          ENABLE_TELEMETRY.updateValue<boolean>(result, ConfigurationTarget.Global),
+        ]);
+      }
+    }
+  }
+
+  /**
+   * Exposed for testing
+   */
+  get _reporter() {
+    return this.reporter;
+  }
+
+  private disposeReporter() {
+    if (this.reporter) {
+      this.reporter.dispose();
+      this.reporter = undefined;
+    }
+  }
+
+  private wasTelemetryRequested(): boolean {
+    return !!this.ctx.globalState.get<boolean>('telemetry-request-viewed');
+  }
+
+  private async setTelemetryRequested(newValue: boolean): Promise<void> {
+    await this.ctx.globalState.update('telemetry-request-viewed', newValue);
+  }
+}
+
+
+/**
+ * The global Telemetry instance
+ */
+export let telemetryListener: TelemetryListener;
+
+export async function initializeTelemetry(extension: Extension<any>, ctx: ExtensionContext): Promise<void> {
+  telemetryListener = new TelemetryListener(extension.id, extension.packageJSON.version, key, ctx);
+  telemetryListener.initialize();
+  ctx.subscriptions.push(telemetryListener);
+}
--- a/extensions/ql-vscode/src/upgrades.ts
+++ b/extensions/ql-vscode/src/upgrades.ts
@@ -1,6 +1,5 @@
 import * as vscode from 'vscode';
-import { DatabaseItem } from './databases';
-import { showAndLogErrorMessage } from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
 import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 import * as messages from './pure/messages';
@@ -9,7 +8,6 @@ import { upgradesTmpDir } from './run-queries';
 import * as tmp from 'tmp-promise';
 import * as path from 'path';
 import * as semver from 'semver';
-import { getOnDiskWorkspaceFolders } from './helpers';

 /**
 * Maximum number of lines to include from database upgrade message,
--- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts
@@ -21,17 +21,19 @@ describe('databaseFetcher', function() {
  this.timeout(10000);

  describe('convertToDatabaseUrl', () => {
+    let sandbox: sinon.SinonSandbox;
    let quickPickSpy: sinon.SinonStub;
    beforeEach(() => {
-      quickPickSpy = sinon.stub(window, 'showQuickPick');
+      sandbox = sinon.createSandbox();
+      quickPickSpy = sandbox.stub(window, 'showQuickPick');
    });

    afterEach(() => {
-      (window.showQuickPick as sinon.SinonStub).restore();
+      sandbox.restore();
    });

    it('should convert a project url to a database url', async () => {
-      quickPickSpy.returns('javascript' as any);
+      quickPickSpy.resolves('javascript');
      const lgtmUrl = 'https://lgtm.com/projects/g/github/codeql';
      const dbUrl = await convertToDatabaseUrl(lgtmUrl);

@@ -43,7 +45,7 @@ describe('databaseFetcher', function() {
    });

    it('should convert a project url to a database url with extra path segments', async () => {
-      quickPickSpy.returns('python' as any);
+      quickPickSpy.resolves('python');
      const lgtmUrl =
        'https://lgtm.com/projects/g/github/codeql/subpage/subpage2?query=xxx';
      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
@@ -54,7 +56,7 @@ describe('databaseFetcher', function() {
    });

    it('should fail on a nonexistant prohect', async () => {
-      quickPickSpy.returns('javascript' as any);
+      quickPickSpy.resolves('javascript');
      const lgtmUrl = 'https://lgtm.com/projects/g/github/hucairz';
      expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
    });
--- a/extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts
@@ -190,10 +190,11 @@ describe('Launcher path', () => {
  let launcherThatExists = '';

  beforeEach(() => {
+    sandbox = sinon.createSandbox();
    getExecutableFromDirectory = createModule().getExecutableFromDirectory;
  });

-  beforeEach(() => {
+  afterEach(() => {
    sandbox.restore();
  });

@@ -300,7 +301,6 @@ describe('Launcher path', () => {
  });

  function createModule() {
-    sandbox = sinon.createSandbox();
    warnSpy = sandbox.spy();
    errorSpy = sandbox.spy();
    logSpy = sandbox.spy();
--- a/extensions/ql-vscode/src/vscode-tests/no-workspace/telemetry.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/telemetry.test.ts
@@ -0,0 +1,380 @@
+import * as chai from 'chai';
+import 'mocha';
+import 'sinon-chai';
+import * as sinon from 'sinon';
+import * as chaiAsPromised from 'chai-as-promised';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ExtensionContext, workspace, ConfigurationTarget, window } from 'vscode';
+import { TelemetryListener, telemetryListener as globalTelemetryListener } from '../../telemetry';
+import { UserCancellationException } from '../../commandRunner';
+import { fail } from 'assert';
+import { ENABLE_TELEMETRY } from '../../config';
+
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
+const sandbox = sinon.createSandbox();
+
+describe('telemetry reporting', function() {
+  // setting preferences can trigger lots of background activity
+  // so need to bump up the timeout of this test.
+  this.timeout(10000);
+
+  let originalTelemetryExtension: boolean | undefined;
+  let originalTelemetryGlobal: boolean | undefined;
+  let isCanary: string;
+  let ctx: ExtensionContext;
+  let telemetryListener: TelemetryListener;
+
+  beforeEach(async () => {
+    try {
+      // in case a previous test has accidentally activated this extension,
+      // need to disable it first.
+      // Accidentaly activation may happen asynchronously due to activationEvents
+      // specified in the package.json.
+      globalTelemetryListener?.dispose();
+
+      ctx = createMockExtensionContext();
+
+      sandbox.stub(TelemetryReporter.prototype, 'sendTelemetryEvent');
+      sandbox.stub(TelemetryReporter.prototype, 'sendTelemetryException');
+      sandbox.stub(TelemetryReporter.prototype, 'dispose');
+
+      originalTelemetryExtension = workspace.getConfiguration().get<boolean>('codeQL.telemetry.enableTelemetry');
+      originalTelemetryGlobal = workspace.getConfiguration().get<boolean>('telemetry.enableTelemetry');
+      isCanary = (!!workspace.getConfiguration().get<boolean>('codeQL.canary')).toString();
+
+      // each test will default to telemetry being enabled
+      await enableTelemetry('telemetry', true);
+      await enableTelemetry('codeQL.telemetry', true);
+
+      telemetryListener = new TelemetryListener('my-id', '1.2.3', 'fake-key', ctx);
+      await wait(100);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+
+  afterEach(async () => {
+    telemetryListener?.dispose();
+    // await wait(100);
+    try {
+      sandbox.restore();
+      await enableTelemetry('telemetry', originalTelemetryGlobal);
+      await enableTelemetry('codeQL.telemetry', originalTelemetryExtension);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+
+  it('should initialize telemetry when both options are enabled', async () => {
+    await telemetryListener.initialize();
+
+    expect(telemetryListener._reporter).not.to.be.undefined;
+
+    const reporter: any = telemetryListener._reporter;
+    expect(reporter.extensionId).to.eq('my-id');
+    expect(reporter.extensionVersion).to.eq('1.2.3');
+    expect(reporter.userOptIn).to.eq(true); // enabled
+  });
+
+  it('should initialize telemetry when global option disabled', async () => {
+    try {
+      await enableTelemetry('telemetry', false);
+      await telemetryListener.initialize();
+      expect(telemetryListener._reporter).not.to.be.undefined;
+
+      const reporter: any = telemetryListener._reporter;
+      expect(reporter.userOptIn).to.eq(false); // disabled
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should not initialize telemetry when extension option disabled', async () => {
+    try {
+      await enableTelemetry('codeQL.telemetry', false);
+      await telemetryListener.initialize();
+
+      expect(telemetryListener._reporter).to.be.undefined;
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should not initialize telemetry when both options disabled', async () => {
+    await enableTelemetry('codeQL.telemetry', false);
+    await enableTelemetry('telemetry', false);
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).to.be.undefined;
+  });
+
+  it('should dispose telemetry object when re-initializing and should not add multiple', async () => {
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    const firstReporter = telemetryListener._reporter;
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    expect(telemetryListener._reporter).not.to.eq(firstReporter);
+
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+
+    // initializing a third time continues to dispose
+    await telemetryListener.initialize();
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledTwice;
+  });
+
+  it('should reinitialize reporter when extension setting changes', async () => {
+    await telemetryListener.initialize();
+
+    expect(TelemetryReporter.prototype.dispose).not.to.have.been.called;
+    expect(telemetryListener._reporter).not.to.be.undefined;
+
+    // this disables the reporter
+    await enableTelemetry('codeQL.telemetry', false);
+
+    expect(telemetryListener._reporter).to.be.undefined;
+
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+
+    // creates a new reporter, but does not dispose again
+    await enableTelemetry('codeQL.telemetry', true);
+
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+  });
+
+  it('should set userOprIn to false when global setting changes', async () => {
+    await telemetryListener.initialize();
+
+    const reporter: any = telemetryListener._reporter;
+    expect(reporter.userOptIn).to.eq(true); // enabled
+
+    await enableTelemetry('telemetry', false);
+    expect(reporter.userOptIn).to.eq(false); // disabled
+  });
+
+  it('should send an event', async () => {
+    await telemetryListener.initialize();
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Success',
+        isCanary
+      },
+      { executionTime: 1234 });
+
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should send a command usage event with an error', async () => {
+    await telemetryListener.initialize();
+
+    telemetryListener.sendCommandUsage('command-id', 1234, new UserCancellationException());
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Cancelled',
+        isCanary
+      },
+      { executionTime: 1234 });
+
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should avoid sending an event when telemetry is disabled', async () => {
+    await telemetryListener.initialize();
+    await enableTelemetry('codeQL.telemetry', false);
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+    telemetryListener.sendCommandUsage('command-id', 1234, new Error());
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).not.to.have.been.called;
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should send an event when telemetry is re-enabled', async () => {
+    await telemetryListener.initialize();
+    await enableTelemetry('codeQL.telemetry', false);
+    await enableTelemetry('codeQL.telemetry', true);
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Success',
+        isCanary
+      },
+      { executionTime: 1234 });
+  });
+
+  it('should filter undesired properties from telemetry payload', async () => {
+    await telemetryListener.initialize();
+    // Reach into the internal appInsights client to grab our telemetry processor.
+    const telemetryProcessor: Function =
+      ((telemetryListener._reporter as any).appInsightsClient._telemetryProcessors)[0];
+    const envelop = {
+      tags: {
+        'ai.cloud.roleInstance': true,
+        other: true
+      },
+      data: {
+        baseData: {
+          properties: {
+            'common.remotename': true,
+            other: true
+          }
+        }
+      }
+    };
+    const res = telemetryProcessor(envelop);
+    expect(res).to.eq(true);
+    expect(envelop).to.deep.eq({
+      tags: {
+        other: true
+      },
+      data: {
+        baseData: {
+          properties: {
+            other: true
+          }
+        }
+      }
+    });
+  });
+
+  it('should request permission if popup has never been seen before', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolvesArg(2 /* "yes" item */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    await enableTelemetry('codeQL.telemetry', false);
+
+    await telemetryListener.initialize();
+
+    // Dialog opened, user clicks "yes" and telemetry enabled
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(true);
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.true;
+  });
+
+  it('should prevent telemetry if permission is denied', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolvesArg(3 /* "no" item */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    await enableTelemetry('codeQL.telemetry', true);
+
+    await telemetryListener.initialize();
+
+    // Dialog opened, user clicks "no" and telemetry disabled
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(false);
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.true;
+  });
+
+  it('should unchange telemetry if permission dialog is dismissed', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+
+    // this causes requestTelemetryPermission to be called
+    await enableTelemetry('codeQL.telemetry', false);
+
+    // Dialog opened, and user closes without interacting with it
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(false);
+    // dialog was canceled, so should not have marked as viewed
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+  });
+
+  it('should unchange telemetry if permission dialog is cancelled if starting as true', async () => {
+    await enableTelemetry('codeQL.telemetry', false);
+
+    // as before, except start with telemetry enabled. It should _stay_ enabled if the
+    // dialog is canceled.
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+
+    // this causes requestTelemetryPermission to be called
+    await enableTelemetry('codeQL.telemetry', true);
+
+    // Dialog opened, and user closes without interacting with it
+    // Telemetry state should not have changed
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(true);
+    // dialog was canceled, so should not have marked as viewed
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+  });
+
+  it('should avoid showing dialog if global telemetry is disabled', async () => {
+    // when telemetry is disabled globally, we never want to show the
+    // opt in/out dialog. We just assume that codeql telemetry should
+    // remain disabled as well.
+    // If the user ever turns global telemetry back on, then we can
+    // show the dialog.
+
+    await enableTelemetry('telemetry', false);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    sandbox.stub(window, 'showInformationMessage');
+
+    await telemetryListener.initialize();
+
+    // popup should not be shown even though we have initialized telemetry
+    expect(window.showInformationMessage).not.to.have.been.called;
+  });
+
+  // This test is failing because codeQL.canary is not a registered configuration.
+  // We do not want to have it registered because we don't want this item
+  // appearing in the settings page. It needs to olny be set by users we tell
+  // about it to.
+  // At this point, I see no other way of testing re-requesting permission.
+  xit('should request permission again when user changes canary setting', async () => {
+    // initially, both canary and telemetry are false
+    await workspace.getConfiguration().update('codeQL.canary', false);
+    await enableTelemetry('codeQL.telemetry', false);
+    await ctx.globalState.update('telemetry-request-viewed', true);
+    await telemetryListener.initialize();
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+
+    // set canary to true
+    await workspace.getConfiguration().update('codeQL.canary', true);
+
+    // now, we should have to click through the telemetry requestor again
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+  });
+
+  function createMockExtensionContext(): ExtensionContext {
+    return {
+      globalState: {
+        _state: {
+          'telemetry-request-viewed': true
+        } as Record<string, any>,
+        get(key: string) {
+          return this._state[key];
+        },
+        update(key: string, val: any) {
+          this._state[key] = val;
+        }
+      }
+    } as any;
+  }
+
+  async function enableTelemetry(section: string, value: boolean | undefined) {
+    await workspace.getConfiguration(section).update(
+      'enableTelemetry', value, ConfigurationTarget.Global
+    );
+
+    // Need to wait some time since the onDidChangeConfiguration listeners fire
+    // asynchronously and we sometimes need to wait for them to complete in
+    // order to have as successful test.
+    await wait(50);
+  }
+
+  async function wait(ms = 0) {
+    return new Promise(resolve =>
+      setTimeout(resolve, ms)
+    );
+  }
+});