Merge branch 'main' into aeisenberg/repo-filter

extensions/ql-vscode/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## [UNRELEASED]
 
 - Fix a bug where the AST viewer was not synchronizing its selected node when the editor selection changes. [#1230](https://github.com/github/vscode-codeql/pull/1230)
+- Open the directory in the finder/explorer (instead of just highlighting it) when running the "Open query directory" command from the query history view. [#1235](https://github.com/github/vscode-codeql/pull/1235)
+- Ensure query label in the query history view changes are persisted across restarts. [#1235](https://github.com/github/vscode-codeql/pull/1235)
 
 ## 1.6.1 - 17 March 2022
 
@@ -15,6 +17,7 @@ No user facing changes.
 - Fix a bug where queries took a long time to run if there are no folders in the workspace. [#1157](https://github.com/github/vscode-codeql/pull/1157)
 - [BREAKING CHANGE] The `codeQL.runningQueries.customLogDirectory` setting is deprecated and no longer has any function. Instead, all query log files will be stored in the query history directory, next to the query results. [#1178](https://github.com/github/vscode-codeql/pull/1178)
 - Add a _Open query directory_ command for query items. This command opens the directory containing all artifacts for a query. [#1179](https://github.com/github/vscode-codeql/pull/1179)
+- Add options to display evaluator logs for a given query run. Some information that was previously found in the query server output may now be found here. [#1186](https://github.com/github/vscode-codeql/pull/1186)
 
 ## 1.5.11 - 10 February 2022
 

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -7,8 +7,7 @@ export function compileView(cb: (err?: Error) => void) {
 
 export function watchView(cb: (err?: Error) => void) {
   const watchConfig = {
-    ...
+    ...config,
-    config,
     watch: true,
     watchOptions: {
       aggregateTimeout: 200,

extensions/ql-vscode/package.json

@@ -522,6 +522,14 @@
         "command": "codeQLQueryHistory.openQueryDirectory",
         "title": "Open query directory"
       },
+      {
+        "command": "codeQLQueryHistory.showEvalLog",
+        "title": "Show Evaluator Log (Raw)"
+      },
+      {
+        "command": "codeQLQueryHistory.showEvalLogSummary",
+        "title": "Show Evaluator Log (Summary)"
+      },
       {
         "command": "codeQLQueryHistory.cancel",
         "title": "Cancel"
@@ -725,6 +733,16 @@
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && !hasRemoteServer"
         },
+        {
+          "command": "codeQLQueryHistory.showEvalLog",
+          "group": "9_qlCommands",
+          "when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogSummary",
+          "group": "9_qlCommands",
+          "when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)"
+        },
         {
           "command": "codeQLQueryHistory.showQueryText",
           "group": "9_qlCommands",
@@ -924,6 +942,14 @@
           "command": "codeQLQueryHistory.showQueryLog",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.showEvalLog",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogSummary",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.openQueryDirectory",
           "when": "false"
@@ -1091,7 +1117,7 @@
     "classnames": "~2.2.6",
     "d3": "^6.3.1",
     "d3-graphviz": "^2.6.1",
-    "fs-extra": "^9.0.1",
+    "fs-extra": "^10.0.1",
     "glob-promise": "^3.4.0",
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.6",
@@ -1149,7 +1175,7 @@
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
     "@types/vscode": "^1.59.0",
-    "@types/webpack": "^4.32.1",
+    "@types/webpack": "^5.28.0",
     "@types/xml2js": "~0.4.4",
     "@typescript-eslint/eslint-plugin": "^4.26.0",
     "@typescript-eslint/parser": "^4.26.0",
@@ -1177,13 +1203,13 @@
     "sinon": "~9.0.0",
     "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
-    "through2": "^3.0.1",
+    "through2": "^4.0.2",
     "ts-loader": "^8.1.0",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
     "typescript": "^4.5.5",
     "typescript-formatter": "^7.2.2",
-    "vsce": "^1.65.0",
+    "vsce": "^2.7.0",
     "vscode-test": "^1.4.0",
     "webpack": "^5.28.0",
     "webpack-cli": "^4.6.0"

extensions/ql-vscode/src/cli.ts

@@ -8,7 +8,7 @@ import { Readable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as tk from 'tree-kill';
 import { promisify } from 'util';
-import { CancellationToken, Disposable, Uri } from 'vscode';
+import { CancellationToken, commands, Disposable, Uri } from 'vscode';
 
 import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
 import { CliConfig } from './config';
@@ -665,6 +665,23 @@ export class CodeQLCliServer implements Disposable {
     return await this.runCodeQlCliCommand(['generate', 'query-help'], subcommandArgs, `Generating qhelp in markdown format at ${outputDirectory}`);
   }
 
+  /**
+  * Generate a summary of an evaluation log.
+  * @param inputPath The path of an evaluation event log.
+  * @param outputPath The path to write a human-readable summary of it to.
+  */
+   async generateLogSummary(
+    inputPath: string,
+    outputPath: string,
+  ): Promise<string> {
+    const subcommandArgs = [
+      '--format=text',
+      inputPath,
+      outputPath
+    ];
+    return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
@@ -940,6 +957,10 @@ export class CodeQLCliServer implements Disposable {
   public async getVersion() {
     if (!this._version) {
       this._version = await this.refreshVersion();
+      // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
+      await commands.executeCommand(
+        'setContext', 'codeql.supportsEvalLog', await this.cliConstraints.supportsPerQueryEvalLog()
+      );
     }
     return this._version;
   }
@@ -1256,6 +1277,11 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
 
+   /**
+    * CLI version that supports rotating structured logs to produce one per query.
+    */
+    public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.8.4');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1315,4 +1341,8 @@ export class CliVersionConstraint {
   async supportsStructuredEvalLog() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG);
   }
+
+  async supportsPerQueryEvalLog() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG);
+  }
 }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -86,15 +86,15 @@ export async function promptImportGithubDatabase(
     maxStep: 2
   });
   const githubRepo = await window.showInputBox({
-    title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
+    title: 'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
-    placeHolder: '<owner>/<repo>',
+    placeHolder: 'https://github.com/<owner>/<repo> or <owner>/<repo>',
     ignoreFocusOut: true,
   });
   if (!githubRepo) {
     return;
   }
 
-  if (!REPO_REGEX.test(githubRepo)) {
+  if (!looksLikeGithubRepo(githubRepo)) {
     throw new Error(`Invalid GitHub repository: ${githubRepo}`);
   }
 
@@ -465,18 +465,62 @@ export async function findDirWithFile(
   return;
 }
 
+/**
+ * The URL pattern is https://github.com/{owner}/{name}/{subpages}.
+ *
+ * This function accepts any URL that matches the pattern above. It also accepts just the
+ * name with owner (NWO): `<owner>/<repo>`.
+ *
+ * @param githubRepo The GitHub repository URL or NWO
+ *
+ * @return true if this looks like a valid GitHub repository URL or NWO
+ */
+export function looksLikeGithubRepo(
+  githubRepo: string | undefined
+): githubRepo is string {
+  if (!githubRepo) {
+    return false;
+  }
+  if (REPO_REGEX.test(githubRepo) || convertGitHubUrlToNwo(githubRepo)) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Converts a GitHub repository URL to the corresponding NWO.
+ * @param githubUrl The GitHub repository URL
+ * @return The corresponding NWO, or undefined if the URL is not valid
+ */
+function convertGitHubUrlToNwo(githubUrl: string): string | undefined {
+  try {
+    const uri = Uri.parse(githubUrl, true);
+    if (uri.scheme !== 'https') {
+      return;
+    }
+    if (uri.authority !== 'github.com' && uri.authority !== 'www.github.com') {
+      return;
+    }
+    const paths = uri.path.split('/').filter((segment: string) => segment);
+    const nwo = `${paths[0]}/${paths[1]}`;
+    if (REPO_REGEX.test(nwo)) {
+      return nwo;
+    }
+    return;
+  } catch (e) {
+    // Ignore the error here, since we catch failures at a higher level.
+    // In particular: returning undefined leads to an error in 'promptImportGithubDatabase'.
+    return;
+  }
+}
+
 export async function convertGithubNwoToDatabaseUrl(
   githubRepo: string,
   credentials: Credentials,
   progress: ProgressCallback): Promise<string | undefined> {
   try {
-    // TODO: In future, we could accept GitHub URLs in addition to NWOs.
+    const nwo = convertGitHubUrlToNwo(githubRepo) || githubRepo;
-    // Similar to "looksLikeLgtmUrl".
+    const [owner, repo] = nwo.split('/');
-    if (!REPO_REGEX.test(githubRepo)) {
-      throw new Error('Invalid repository format. Must be in the format <owner>/<repo> (e.g. github/codeql)');
-    }
-
-    const [owner, repo] = githubRepo.split('/');
 
     const octokit = await credentials.getOctokit();
     const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo });
@@ -531,7 +575,7 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
       return false;
     }
 
-    const paths = uri.path.split('/').filter((segment) => segment);
+    const paths = uri.path.split('/').filter((segment: string) => segment);
     return paths.length >= 4 && paths[0] === 'projects';
   } catch (e) {
     return false;
@@ -604,7 +648,7 @@ export async function convertLgtmUrlToDatabaseUrl(
 async function downloadLgtmProjectMetadata(lgtmUrl: string): Promise<any> {
   const uri = Uri.parse(lgtmUrl, true);
   const paths = ['api', 'v1.0'].concat(
-    uri.path.split('/').filter((segment) => segment)
+    uri.path.split('/').filter((segment: string) => segment)
   ).slice(0, 6);
   const projectUrl = `https://lgtm.com/${paths.join('/')}`;
   const projectResponse = await fetch(projectUrl);

extensions/ql-vscode/src/extension.ts

@@ -535,6 +535,8 @@ async function activateWithInstalledDistribution(
           queryStorageDir,
           progress,
           source.token,
+          undefined,
+          item,
         );
         item.completeThisQuery(completedQueryInfo);
         await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced);

extensions/ql-vscode/src/pure/messages.ts

@@ -646,6 +646,35 @@ export interface ClearCacheParams {
    */
   dryRun: boolean;
 }
+
+/**
+ * Parameters to start a new structured log
+ */
+ export interface StartLogParams {
+  /**
+   * The dataset for which we want to start a new structured log
+   */
+  db: Dataset;
+  /**
+   * The path where we want to place the new structured log
+   */
+  logPath: string;
+}
+
+/**
+ * Parameters to terminate a structured log
+ */
+ export interface EndLogParams {
+  /**
+   * The dataset for which we want to terminated the log
+   */
+  db: Dataset;
+  /**
+   * The path of the log to terminate, will be a no-op if we aren't logging here
+   */
+  logPath: string;
+}
+
 /**
  * Parameters for trimming the cache of a dataset
  */
@@ -682,6 +711,26 @@ export interface ClearCacheResult {
   deletionMessage: string;
 }
 
+/**
+ * The result of starting a new structured log.
+ */
+export interface StartLogResult {
+  /**
+   * A user friendly message saying what happened.
+   */
+  outcomeMessage: string;
+}
+
+/**
+ * The result of terminating a structured.
+ */
+export interface EndLogResult {
+  /**
+   * A user friendly message saying what happened.
+   */
+  outcomeMessage: string;
+}
+
 /**
  * Parameters for running a set of queries
  */
@@ -1018,6 +1067,16 @@ export const compileUpgrade = new rpc.RequestType<WithProgressId<CompileUpgradeP
  */
 export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<CompileUpgradeSequenceParams>, CompileUpgradeSequenceResult, void, void>('compilation/compileUpgradeSequence');
 
+/**
+ * Start a new structured log in the evaluator, terminating the previous one if it exists
+ */
+ export const startLog = new rpc.RequestType<WithProgressId<StartLogParams>, StartLogResult, void, void>('evaluation/startLog');
+
+/**
+ * Terminate a structured log in the evaluator. Is a no-op if we aren't logging to the given location
+ */
+ export const endLog = new rpc.RequestType<WithProgressId<EndLogParams>, EndLogResult, void, void>('evaluation/endLog');
+
 /**
  * Clear the cache of a dataset
  */

extensions/ql-vscode/src/query-history.ts

@@ -28,12 +28,14 @@ import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
 import { DisposableObject } from './pure/disposable-object';
 import { commandRunner } from './commandRunner';
-import { assertNever, ONE_HOUR_IN_MS, TWO_HOURS_IN_MS, getErrorMessage, getErrorStack  } from './pure/helpers-pure';
+import { assertNever, ONE_HOUR_IN_MS, TWO_HOURS_IN_MS, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { CompletedLocalQueryInfo, LocalQueryInfo as LocalQueryInfo, QueryHistoryInfo } from './query-results';
 import { DatabaseManager } from './databases';
 import { registerQueryHistoryScubber } from './query-history-scrubber';
 import { QueryStatus } from './query-status';
 import { slurpQueryHistory, splatQueryHistory } from './query-serialization';
+import * as fs from 'fs-extra';
+import { CliVersionConstraint } from './cli';
 
 /**
  * query-history.ts
@@ -181,38 +183,48 @@ export class HistoryTreeDataProvider extends DisposableObject {
   ): ProviderResult<QueryHistoryInfo[]> {
     return element ? [] : this.history.sort((h1, h2) => {
 
-      // TODO remote queries are not implemented yet.
+      const h1Label = h1.label.toLowerCase();
-      if (h1.t !== 'local' && h2.t !== 'local') {
+      const h2Label = h2.label.toLowerCase();
-        return 0;
-      }
-      if (h1.t !== 'local') {
-        return -1;
-      }
-      if (h2.t !== 'local') {
-        return 1;
-      }
 
-      const resultCount1 = h1.completedQuery?.resultCount ?? -1;
+      const h1Date = h1.t === 'local'
-      const resultCount2 = h2.completedQuery?.resultCount ?? -1;
+        ? h1.initialInfo.start.getTime()
+        : h1.remoteQuery?.executionStartTime;
+
+      const h2Date = h2.t === 'local'
+        ? h2.initialInfo.start.getTime()
+        : h2.remoteQuery?.executionStartTime;
+
+      // result count for remote queries is not available here.
+      const resultCount1 = h1.t === 'local'
+        ? h1.completedQuery?.resultCount ?? -1
+        : -1;
+      const resultCount2 = h2.t === 'local'
+        ? h2.completedQuery?.resultCount ?? -1
+        : -1;
 
       switch (this.sortOrder) {
         case SortOrder.NameAsc:
-          return h1.label.localeCompare(h2.label, env.language);
+          return h1Label.localeCompare(h2Label, env.language);
+
         case SortOrder.NameDesc:
-          return h2.label.localeCompare(h1.label, env.language);
+          return h2Label.localeCompare(h1Label, env.language);
+
         case SortOrder.DateAsc:
-          return h1.initialInfo.start.getTime() - h2.initialInfo.start.getTime();
+          return h1Date - h2Date;
+
         case SortOrder.DateDesc:
-          return h2.initialInfo.start.getTime() - h1.initialInfo.start.getTime();
+          return h2Date - h1Date;
+
         case SortOrder.CountAsc:
           // If the result counts are equal, sort by name.
           return resultCount1 - resultCount2 === 0
-            ? h1.label.localeCompare(h2.label, env.language)
+            ? h1Label.localeCompare(h2Label, env.language)
             : resultCount1 - resultCount2;
+
         case SortOrder.CountDesc:
           // If the result counts are equal, sort by name.
           return resultCount2 - resultCount1 === 0
-            ? h2.label.localeCompare(h1.label, env.language)
+            ? h2Label.localeCompare(h1Label, env.language)
             : resultCount2 - resultCount1;
         default:
           assertNever(this.sortOrder);
@@ -406,6 +418,18 @@ export class QueryHistoryManager extends DisposableObject {
         this.handleOpenQueryDirectory.bind(this)
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.showEvalLog',
+        this.handleShowEvalLog.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.showEvalLogSummary',
+        this.handleShowEvalLogSummary.bind(this)
+      )
+    );
     this.push(
       commandRunner(
         'codeQLQueryHistory.cancel',
@@ -636,7 +660,7 @@ export class QueryHistoryManager extends DisposableObject {
     if (response !== undefined) {
       // Interpret empty string response as 'go back to using default'
       finalSingleItem.initialInfo.userSpecifiedLabel = response === '' ? undefined : response;
-      this.treeDataProvider.refresh();
+      await this.refreshTreeView();
     }
   }
 
@@ -727,21 +751,72 @@ export class QueryHistoryManager extends DisposableObject {
       return;
     }
 
-    let p: string | undefined;
+    let externalFilePath: string | undefined;
     if (finalSingleItem.t === 'local') {
       if (finalSingleItem.completedQuery) {
-        p = finalSingleItem.completedQuery.query.querySaveDir;
+        externalFilePath = path.join(finalSingleItem.completedQuery.query.querySaveDir, 'timestamp');
       }
     } else if (finalSingleItem.t === 'remote') {
-      p = path.join(this.queryStorageDir, finalSingleItem.queryId);
+      externalFilePath = path.join(this.queryStorageDir, finalSingleItem.queryId, 'timestamp');
     }
 
-    if (p) {
+    if (externalFilePath) {
-      try {
+      if (!(await fs.pathExists(externalFilePath))) {
-        await commands.executeCommand('revealFileInOS', Uri.file(p));
+        // timestamp file is missing (manually deleted?) try selecting the parent folder.
-      } catch (e) {
+        // It's less nice, but at least it will work.
-        throw new Error(`Failed to open ${p}: ${getErrorMessage(e)}`);
+        externalFilePath = path.dirname(externalFilePath);
+        if (!(await fs.pathExists(externalFilePath))) {
+          throw new Error(`Query directory does not exist: ${externalFilePath}`);
+        }
       }
+      try {
+        await commands.executeCommand('revealFileInOS', Uri.file(externalFilePath));
+      } catch (e) {
+        throw new Error(`Failed to open ${externalFilePath}: ${getErrorMessage(e)}`);
+      }
+    }
+  }
+
+  private warnNoEvalLog() {
+    void showAndLogWarningMessage('No evaluator log is available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG + '?');
+  }
+
+  async handleShowEvalLog(
+    singleItem: QueryHistoryInfo,
+    multiSelect: QueryHistoryInfo[]
+  ) {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+
+    // Only applicable to an individual local query
+    if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') {
+      return;
+    }
+
+    if (finalSingleItem.evalLogLocation) {
+      await this.tryOpenExternalFile(finalSingleItem.evalLogLocation);
+    } else {
+      this.warnNoEvalLog();
+    }
+  }
+
+  async handleShowEvalLogSummary(
+    singleItem: QueryHistoryInfo,
+    multiSelect: QueryHistoryInfo[]
+  ) {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+
+    // Only applicable to an individual local query
+    if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') {
+      return;
+    }
+
+    if (finalSingleItem.evalLogLocation) {
+      if (!fs.existsSync(finalSingleItem.evalLogSummaryLocation)) {
+        await this.qs.cliServer.generateLogSummary(finalSingleItem.evalLogLocation, finalSingleItem.evalLogSummaryLocation);
+      }
+      await this.tryOpenExternalFile(finalSingleItem.evalLogSummaryLocation);
+    } else {
+      this.warnNoEvalLog();
     }
   }
 

extensions/ql-vscode/src/query-results.ts

@@ -216,6 +216,7 @@ export class LocalQueryInfo {
 
   public failureReason: string | undefined;
   public completedQuery: CompletedQueryInfo | undefined;
+  public evalLogLocation: string | undefined;
   private config: QueryHistoryConfig | undefined;
 
   /**
@@ -311,6 +312,14 @@ export class LocalQueryInfo {
     }
   }
 
+  /**
+   * Return the location of a query's evaluator log summary. This file may not exist yet,
+   * in which case it can be created by invoking `codeql generate log-summary`.
+   */
+  get evalLogSummaryLocation(): string {
+    return this.evalLogLocation + '.summary';
+  }
+
   get completed(): boolean {
     return !!this.completedQuery;
   }

extensions/ql-vscode/src/queryserver-client.ts

@@ -146,7 +146,7 @@ export class QueryServerClient extends DisposableObject {
       args.push('--require-db-registration');
     }
 
-    if (await this.cliServer.cliConstraints.supportsOldEvalStats()) {
+    if (await this.cliServer.cliConstraints.supportsOldEvalStats() && !(await this.cliServer.cliConstraints.supportsPerQueryEvalLog())) {
       args.push('--old-eval-stats');
     }
 
@@ -258,3 +258,7 @@ export class QueryServerClient extends DisposableObject {
 export function findQueryLogFile(resultPath: string): string {
   return path.join(resultPath, 'query.log');
 }
+
+export function findQueryStructLogFile(resultPath: string): string {
+  return path.join(resultPath, 'evaluator-log.jsonl');
+}

extensions/ql-vscode/src/run-queries.ts

@@ -29,7 +29,7 @@ import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
 import { logger } from './logging';
 import * as messages from './pure/messages';
-import { InitialQueryInfo } from './query-results';
+import { InitialQueryInfo, LocalQueryInfo } from './query-results';
 import * as qsClient from './queryserver-client';
 import { isQuickQueryPath } from './quick-query';
 import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
@@ -95,6 +95,10 @@ export class QueryEvaluationInfo {
     return qsClient.findQueryLogFile(this.querySaveDir);
   }
 
+  get structLogPath() {
+    return qsClient.findQueryStructLogFile(this.querySaveDir);
+  }
+
   get resultsPaths() {
     return {
       resultsPath: path.join(this.querySaveDir, 'results.bqrs'),
@@ -125,6 +129,7 @@ export class QueryEvaluationInfo {
     dbItem: DatabaseItem,
     progress: ProgressCallback,
     token: CancellationToken,
+    queryInfo?: LocalQueryInfo,
   ): Promise<messages.EvaluationResult> {
     if (!dbItem.contents || dbItem.error) {
       throw new Error('Can\'t run query on invalid database.');
@@ -156,6 +161,12 @@ export class QueryEvaluationInfo {
       dbDir: dbItem.contents.datasetUri.fsPath,
       workingSet: 'default'
     };
+    if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
+      await qs.sendRequest(messages.startLog, {
+        db: dataset,
+        logPath: this.structLogPath,
+      });
+    }
     const params: messages.EvaluateQueriesParams = {
       db: dataset,
       evaluateId: callbackId,
@@ -172,6 +183,13 @@ export class QueryEvaluationInfo {
       }
     } finally {
       qs.unRegisterCallback(callbackId);
+      if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
+        await qs.sendRequest(messages.endLog, {
+          db: dataset,
+          logPath: this.structLogPath,
+        });
+        queryInfo.evalLogLocation = this.structLogPath;
+      }
     }
     return result || {
       evaluationTime: 0,
@@ -658,6 +676,7 @@ export async function compileAndRunQueryAgainstDatabase(
   progress: ProgressCallback,
   token: CancellationToken,
   templates?: messages.TemplateDefinitions,
+  queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
 ): Promise<QueryWithResults> {
   if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
     throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
@@ -743,7 +762,7 @@ export async function compileAndRunQueryAgainstDatabase(
     }
 
     if (errors.length === 0) {
-      const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token);
+      const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
       if (result.resultType !== messages.QueryResultType.SUCCESS) {
         const message = result.message || 'Failed to run query';
         void logger.log(message);

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -12,6 +12,7 @@ import {
   convertLgtmUrlToDatabaseUrl,
   looksLikeLgtmUrl,
   findDirWithFile,
+  looksLikeGithubRepo,
 } from '../../databaseFetcher';
 import { ProgressCallback } from '../../commandRunner';
 import * as pq from 'proxyquire';
@@ -209,6 +210,32 @@ describe('databaseFetcher', function() {
     });
   });
 
+  describe('looksLikeGithubRepo', () => {
+    it('should handle invalid urls', () => {
+      expect(looksLikeGithubRepo(''))
+        .to.be.false;
+      expect(looksLikeGithubRepo('http://github.com/foo/bar'))
+        .to.be.false;
+      expect(looksLikeGithubRepo('https://ww.github.com/foo/bar'))
+        .to.be.false;
+      expect(looksLikeGithubRepo('https://ww.github.com/foo'))
+        .to.be.false;
+      expect(looksLikeGithubRepo('foo'))
+        .to.be.false;
+    });
+
+    it('should handle valid urls', () => {
+      expect(looksLikeGithubRepo('https://github.com/foo/bar'))
+        .to.be.true;
+      expect(looksLikeGithubRepo('https://www.github.com/foo/bar'))
+        .to.be.true;
+      expect(looksLikeGithubRepo('https://github.com/foo/bar/sub/pages'))
+        .to.be.true;
+      expect(looksLikeGithubRepo('foo/bar'))
+        .to.be.true;
+    });
+  });
+
   describe('looksLikeLgtmUrl', () => {
     it('should handle invalid urls', () => {
       expect(looksLikeLgtmUrl('')).to.be.false;

extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts

@@ -427,9 +427,11 @@ describe('query-history', () => {
 
   describe('getChildren', () => {
     const history = [
-      item('a', 10, 20),
+      item('a', 2, 'remote'),
-      item('b', 5, 30),
+      item('b', 10, 'local', 20),
-      item('c', 1, 25),
+      item('c', 5, 'local', 30),
+      item('d', 1, 'local', 25),
+      item('e', 6, 'remote'),
     ];
     let treeDataProvider: HistoryTreeDataProvider;
 
@@ -456,7 +458,7 @@ describe('query-history', () => {
     });
 
     it('should get children for date ascending', async () => {
-      const expected = [history[2], history[1], history[0]];
+      const expected = [history[3], history[0], history[2], history[4], history[1]];
       treeDataProvider.sortOrder = SortOrder.DateAsc;
 
       const children = await treeDataProvider.getChildren();
@@ -464,7 +466,7 @@ describe('query-history', () => {
     });
 
     it('should get children for date descending', async () => {
-      const expected = [history[0], history[1], history[2]];
+      const expected = [history[3], history[0], history[2], history[4], history[1]].reverse();
       treeDataProvider.sortOrder = SortOrder.DateDesc;
 
       const children = await treeDataProvider.getChildren();
@@ -472,7 +474,7 @@ describe('query-history', () => {
     });
 
     it('should get children for result count ascending', async () => {
-      const expected = [history[0], history[2], history[1]];
+      const expected = [history[0], history[4], history[1], history[3], history[2]];
       treeDataProvider.sortOrder = SortOrder.CountAsc;
 
       const children = await treeDataProvider.getChildren();
@@ -480,7 +482,7 @@ describe('query-history', () => {
     });
 
     it('should get children for result count descending', async () => {
-      const expected = [history[1], history[2], history[0]];
+      const expected = [history[0], history[4], history[1], history[3], history[2]].reverse();
       treeDataProvider.sortOrder = SortOrder.CountDesc;
 
       const children = await treeDataProvider.getChildren();
@@ -509,17 +511,27 @@ describe('query-history', () => {
       expect(children).to.deep.eq(expected);
     });
 
-    function item(label: string, start: number, resultCount?: number) {
+    function item(label: string, start: number, t = 'local', resultCount?: number) {
-      return {
+      if (t === 'local') {
-        label,
+        return {
-        initialInfo: {
+          label,
-          start: new Date(start),
+          initialInfo: {
-        },
+            start: new Date(start),
-        completedQuery: {
+          },
-          resultCount,
+          completedQuery: {
-        },
+            resultCount,
-        t: 'local'
+          },
-      };
+          t
+        };
+      } else {
+        return {
+          label,
+          remoteQuery: {
+            executionStartTime: start,
+          },
+          t
+        };
+      }
     }
   });