Merge branch 'main' into koesie10/create-extension-model-file

2023-04-14 10:56:27 +02:00
parent 9261519636 69c66f3b53
commit 370874a02f
13 changed files with 834 additions and 99 deletions
--- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
@@ -194,7 +194,6 @@ export class DataExtensionsEditorView extends AbstractWebview<
        queryRunner: this.queryRunner,
        databaseItem: this.databaseItem,
        queryStorageDir: this.queryStorageDir,
-        logger: extLogger,
        progress: (progressUpdate: ProgressUpdate) => {
          void this.showProgress(progressUpdate, 1500);
        },
--- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
@@ -1,25 +1,27 @@
 import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
-import { qlpackOfDatabase } from "../contextual/queryResolver";
-import { file } from "tmp-promise";
+import { dir } from "tmp-promise";
 import { writeFile } from "fs-extra";
 import { dump as dumpYaml } from "js-yaml";
 import {
  getOnDiskWorkspaceFolders,
  showAndLogExceptionWithTelemetry,
 } from "../helpers";
-import { Logger, TeeLogger } from "../common";
+import { TeeLogger } from "../common";
 import { CancellationToken } from "vscode";
 import { CodeQLCliServer } from "../cli";
 import { DatabaseItem } from "../local-databases";
 import { ProgressCallback } from "../progress";
+import { fetchExternalApiQueries } from "./queries";
+import { QueryResultType } from "../pure/new-messages";
+import { join } from "path";
 import { redactableError } from "../pure/errors";
+import { QueryLanguage } from "../common/query-language";

 export type RunQueryOptions = {
-  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveQueriesInSuite">;
+  cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
  queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
  databaseItem: Pick<DatabaseItem, "contents" | "databaseUri" | "language">;
  queryStorageDir: string;
-  logger: Logger;

  progress: ProgressCallback;
  token: CancellationToken;
@@ -30,54 +32,53 @@ export async function runQuery({
  queryRunner,
  databaseItem,
  queryStorageDir,
-  logger,
  progress,
  token,
 }: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
-  const qlpacks = await qlpackOfDatabase(cliServer, databaseItem);
+  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
+  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
+  // This is intentionally not pretty code, as it will be removed soon.
+  // For a reference of what this should do in the future, see the previous implementation in
+  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72

-  const packsToSearch = [qlpacks.dbschemePack];
-  if (qlpacks.queryPack) {
-    packsToSearch.push(qlpacks.queryPack);
+  const query = fetchExternalApiQueries[databaseItem.language as QueryLanguage];
+  if (!query) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`No external API usage query found for language ${databaseItem.language}`,
+    );
+    return;
  }

-  const suiteFile = (
-    await file({
-      postfix: ".qls",
-    })
-  ).path;
-  const suiteYaml = [];
-  for (const qlpack of packsToSearch) {
-    suiteYaml.push({
-      from: qlpack,
-      queries: ".",
-      include: {
-        id: `${databaseItem.language}/telemetry/fetch-external-apis`,
-      },
-    });
+  const queryDir = (await dir({ unsafeCleanup: true })).path;
+  const queryFile = join(queryDir, "FetchExternalApis.ql");
+  await writeFile(queryFile, query.mainQuery, "utf8");
+
+  if (query.dependencies) {
+    for (const [filename, contents] of Object.entries(query.dependencies)) {
+      const dependencyFile = join(queryDir, filename);
+      await writeFile(dependencyFile, contents, "utf8");
+    }
  }
-  await writeFile(suiteFile, dumpYaml(suiteYaml), "utf8");
+
+  const syntheticQueryPack = {
+    name: "codeql/external-api-usage",
+    version: "0.0.0",
+    dependencies: {
+      [`codeql/${databaseItem.language}-all`]: "*",
+    },
+  };
+
+  const qlpackFile = join(queryDir, "codeql-pack.yml");
+  await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8");

  const additionalPacks = getOnDiskWorkspaceFolders();
  const extensionPacks = Object.keys(
    await cliServer.resolveQlpacks(additionalPacks, true),
  );

-  const queries = await cliServer.resolveQueriesInSuite(
-    suiteFile,
-    getOnDiskWorkspaceFolders(),
-  );
-
-  if (queries.length !== 1) {
-    void logger.log(`Expected exactly one query, got ${queries.length}`);
-    return;
-  }
-
-  const query = queries[0];
-
  const queryRun = queryRunner.createQueryRun(
    databaseItem.databaseUri.fsPath,
-    { queryPath: query, quickEvalPosition: undefined },
+    { queryPath: queryFile, quickEvalPosition: undefined },
    false,
    getOnDiskWorkspaceFolders(),
    extensionPacks,
@@ -86,11 +87,22 @@ export async function runQuery({
    undefined,
  );

-  return queryRun.evaluate(
+  const completedQuery = await queryRun.evaluate(
    progress,
    token,
    new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
  );
+
+  if (completedQuery.resultType !== QueryResultType.SUCCESS) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`External API usage query failed: ${
+        completedQuery.message ?? "No message"
+      }`,
+    );
+    return;
+  }
+
+  return completedQuery;
 }

 export type GetResultsOptions = {
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/index.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/index.ts
@@ -0,0 +1,7 @@
+import { fetchExternalApisQuery as javaFetchExternalApisQuery } from "./java";
+import { Query } from "./query";
+import { QueryLanguage } from "../../common/query-language";
+
+export const fetchExternalApiQueries: Partial<Record<QueryLanguage, Query>> = {
+  [QueryLanguage.Java]: javaFetchExternalApisQuery,
+};
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
@@ -0,0 +1,183 @@
+import { Query } from "./query";
+
+export const fetchExternalApisQuery: Query = {
+  mainQuery: `/**
+ * @name Usage of APIs coming from external libraries
+ * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
+ * @tags telemetry
+ * @id java/telemetry/fetch-external-apis
+ */
+
+import java
+import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+import ExternalApi
+
+private Call aUsage(ExternalApi api) {
+  result.getCallee().getSourceDeclaration() = api and
+  not result.getFile() instanceof GeneratedFile
+}
+
+private boolean isSupported(ExternalApi api) {
+  api.isSupported() and result = true
+  or
+  api = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() and result = true
+  or
+  not api.isSupported() and
+  not api = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() and
+  result = false
+}
+
+from ExternalApi api, string apiName, boolean supported, Call usage
+where
+  apiName = api.getApiName() and
+  supported = isSupported(api) and
+  usage = aUsage(api)
+select apiName, supported, usage
+`,
+  dependencies: {
+    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+
+private import java
+private import semmle.code.java.dataflow.DataFlow
+private import semmle.code.java.dataflow.ExternalFlow
+private import semmle.code.java.dataflow.FlowSources
+private import semmle.code.java.dataflow.FlowSummary
+private import semmle.code.java.dataflow.internal.DataFlowPrivate
+private import semmle.code.java.dataflow.TaintTracking
+
+pragma[nomagic]
+private predicate isTestPackage(Package p) {
+  p.getName()
+      .matches([
+          "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
+          "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
+          "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
+          "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
+          "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
+          "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
+          "org.testng%"
+        ])
+}
+
+/**
+ * A test library.
+ */
+private class TestLibrary extends RefType {
+  TestLibrary() { isTestPackage(this.getPackage()) }
+}
+
+private string containerAsJar(Container container) {
+  if container instanceof JarFile then result = container.getBaseName() else result = "rt.jar"
+}
+
+/** Holds if the given callable is not worth supporting. */
+private predicate isUninteresting(Callable c) {
+  c.getDeclaringType() instanceof TestLibrary or
+  c.(Constructor).isParameterless()
+}
+
+/**
+ * An external API from either the Standard Library or a 3rd party library.
+ */
+class ExternalApi extends Callable {
+  ExternalApi() { not this.fromSource() and not isUninteresting(this) }
+
+  /**
+   * Gets information about the external API in the form expected by the MaD modeling framework.
+   */
+  string getApiName() {
+    result =
+      this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
+        "#" + this.getName() + paramsString(this)
+  }
+
+  /**
+   * Gets the jar file containing this API. Normalizes the Java Runtime to "rt.jar" despite the presence of modules.
+   */
+  string jarContainer() { result = containerAsJar(this.getCompilationUnit().getParentContainer*()) }
+
+  /** Gets a node that is an input to a call to this API. */
+  private DataFlow::Node getAnInput() {
+    exists(Call call | call.getCallee().getSourceDeclaration() = this |
+      result.asExpr().(Argument).getCall() = call or
+      result.(ArgumentNode).getCall().asCall() = call
+    )
+  }
+
+  /** Gets a node that is an output from a call to this API. */
+  private DataFlow::Node getAnOutput() {
+    exists(Call call | call.getCallee().getSourceDeclaration() = this |
+      result.asExpr() = call or
+      result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
+    )
+  }
+
+  /** Holds if this API has a supported summary. */
+  pragma[nomagic]
+  predicate hasSummary() {
+    this = any(SummarizedCallable sc).asCallable() or
+    TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
+  }
+
+  pragma[nomagic]
+  predicate isSource() {
+    this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
+  }
+
+  /** Holds if this API is a known sink. */
+  pragma[nomagic]
+  predicate isSink() { sinkNode(this.getAnInput(), _) }
+
+  /** Holds if this API is supported by existing CodeQL libraries, that is, it is either a recognized source or sink or has a flow summary. */
+  predicate isSupported() { this.hasSummary() or this.isSource() or this.isSink() }
+}
+
+/** DEPRECATED: Alias for ExternalApi */
+deprecated class ExternalAPI = ExternalApi;
+
+/**
+ * Gets the limit for the number of results produced by a telemetry query.
+ */
+int resultLimit() { result = 1000 }
+
+/**
+ * Holds if it is relevant to count usages of \`api\`.
+ */
+signature predicate relevantApi(ExternalApi api);
+
+/**
+ * Given a predicate to count relevant API usages, this module provides a predicate
+ * for restricting the number or returned results based on a certain limit.
+ */
+module Results<relevantApi/1 getRelevantUsages> {
+  private int getUsages(string apiName) {
+    result =
+      strictcount(Call c, ExternalApi api |
+        c.getCallee().getSourceDeclaration() = api and
+        not c.getFile() instanceof GeneratedFile and
+        apiName = api.getApiName() and
+        getRelevantUsages(api)
+      )
+  }
+
+  private int getOrder(string apiInfo) {
+    apiInfo =
+      rank[result](string info, int usages |
+        usages = getUsages(info)
+      |
+        info order by usages desc, info
+      )
+  }
+
+  /**
+   * Holds if there exists an API with \`apiName\` that is being used \`usages\` times
+   * and if it is in the top results (guarded by resultLimit).
+   */
+  predicate restrict(string apiName, int usages) {
+    usages = getUsages(apiName) and
+    getOrder(apiName) <= resultLimit()
+  }
+}
+`,
+  },
+};
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
@@ -0,0 +1,6 @@
+export type Query = {
+  mainQuery: string;
+  dependencies?: {
+    [filename: string]: string;
+  };
+};
--- a/extensions/ql-vscode/src/query-history/store/query-history-domain-mapper.ts
+++ b/extensions/ql-vscode/src/query-history/store/query-history-domain-mapper.ts
@@ -15,20 +15,19 @@ import {
  SortDirectionDto,
 } from "./query-history-local-query-dto";
 import { QueryHistoryItemDto } from "./query-history-dto";
-import { QueryHistoryVariantAnalysisDto } from "./query-history-variant-analysis-dto";
 import {
  RawResultsSortState,
  SortDirection,
  SortedResultSetInfo,
 } from "../../pure/interface-types";
+import { mapQueryHistoryVariantAnalysisToDto } from "./query-history-variant-analysis-domain-mapper";

 export function mapQueryHistoryToDto(
  queries: QueryHistoryInfo[],
 ): QueryHistoryItemDto[] {
  return queries.map((q) => {
    if (q.t === "variant-analysis") {
-      const query: QueryHistoryVariantAnalysisDto = q;
-      return query;
+      return mapQueryHistoryVariantAnalysisToDto(q);
    } else if (q.t === "local") {
      return mapLocalQueryInfoToDto(q);
    } else {
--- a/extensions/ql-vscode/src/query-history/store/query-history-dto-mapper.ts
+++ b/extensions/ql-vscode/src/query-history/store/query-history-dto-mapper.ts
@@ -5,7 +5,6 @@ import {
 } from "../../query-results";
 import { QueryEvaluationInfo } from "../../run-queries-shared";
 import { QueryHistoryInfo } from "../query-history-info";
-import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
 import {
  CompletedQueryInfoDto,
  QueryEvaluationInfoDto,
@@ -23,14 +22,14 @@ import {
  SortDirection,
  SortedResultSetInfo,
 } from "../../pure/interface-types";
+import { mapQueryHistoryVariantAnalysisToDomainModel } from "./query-history-variant-analysis-dto-mapper";

 export function mapQueryHistoryToDomainModel(
  queries: QueryHistoryItemDto[],
 ): QueryHistoryInfo[] {
  return queries.map((d) => {
    if (d.t === "variant-analysis") {
-      const query: VariantAnalysisHistoryItem = d;
-      return query;
+      return mapQueryHistoryVariantAnalysisToDomainModel(d);
    } else if (d.t === "local") {
      return mapLocalQueryItemToDomainModel(d);
    }
--- a/extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-domain-mapper.ts
+++ b/extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-domain-mapper.ts
@@ -0,0 +1,235 @@
+import {
+  QueryHistoryVariantAnalysisDto,
+  QueryLanguageDto,
+  QueryStatusDto,
+  VariantAnalysisDto,
+  VariantAnalysisFailureReasonDto,
+  VariantAnalysisRepoStatusDto,
+  VariantAnalysisScannedRepositoryDto,
+  VariantAnalysisSkippedRepositoriesDto,
+  VariantAnalysisSkippedRepositoryDto,
+  VariantAnalysisSkippedRepositoryGroupDto,
+  VariantAnalysisStatusDto,
+} from "./query-history-variant-analysis-dto";
+import {
+  VariantAnalysis,
+  VariantAnalysisFailureReason,
+  VariantAnalysisRepoStatus,
+  VariantAnalysisScannedRepository,
+  VariantAnalysisSkippedRepositories,
+  VariantAnalysisSkippedRepository,
+  VariantAnalysisSkippedRepositoryGroup,
+  VariantAnalysisStatus,
+} from "../../variant-analysis/shared/variant-analysis";
+import { assertNever } from "../../pure/helpers-pure";
+import { QueryLanguage } from "../../common/query-language";
+import { QueryStatus } from "../../query-status";
+import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
+
+export function mapQueryHistoryVariantAnalysisToDto(
+  item: VariantAnalysisHistoryItem,
+): QueryHistoryVariantAnalysisDto {
+  return {
+    t: "variant-analysis",
+    failureReason: item.failureReason,
+    resultCount: item.resultCount,
+    status: mapQueryStatusToDto(item.status),
+    completed: item.completed,
+    variantAnalysis: mapVariantAnalysisDtoToDto(item.variantAnalysis),
+    userSpecifiedLabel: item.userSpecifiedLabel,
+  };
+}
+
+function mapVariantAnalysisDtoToDto(
+  variantAnalysis: VariantAnalysis,
+): VariantAnalysisDto {
+  return {
+    id: variantAnalysis.id,
+    controllerRepo: {
+      id: variantAnalysis.controllerRepo.id,
+      fullName: variantAnalysis.controllerRepo.fullName,
+      private: variantAnalysis.controllerRepo.private,
+    },
+    query: {
+      name: variantAnalysis.query.name,
+      filePath: variantAnalysis.query.filePath,
+      language: mapQueryLanguageToDto(variantAnalysis.query.language),
+      text: variantAnalysis.query.text,
+    },
+    databases: {
+      repositories: variantAnalysis.databases.repositories,
+      repositoryLists: variantAnalysis.databases.repositoryLists,
+      repositoryOwners: variantAnalysis.databases.repositoryOwners,
+    },
+    createdAt: variantAnalysis.createdAt,
+    updatedAt: variantAnalysis.updatedAt,
+    executionStartTime: variantAnalysis.executionStartTime,
+    status: mapVariantAnalysisStatusToDto(variantAnalysis.status),
+    completedAt: variantAnalysis.completedAt,
+    actionsWorkflowRunId: variantAnalysis.actionsWorkflowRunId,
+    failureReason:
+      variantAnalysis.failureReason &&
+      mapVariantAnalysisFailureReasonToDto(variantAnalysis.failureReason),
+    scannedRepos:
+      variantAnalysis.scannedRepos &&
+      mapVariantAnalysisScannedRepositoriesToDto(variantAnalysis.scannedRepos),
+    skippedRepos:
+      variantAnalysis.skippedRepos &&
+      mapVariantAnalysisSkippedRepositoriesToDto(variantAnalysis.skippedRepos),
+  };
+}
+
+function mapVariantAnalysisScannedRepositoriesToDto(
+  repos: VariantAnalysisScannedRepository[],
+): VariantAnalysisScannedRepositoryDto[] {
+  return repos.map(mapVariantAnalysisScannedRepositoryToDto);
+}
+
+function mapVariantAnalysisScannedRepositoryToDto(
+  repo: VariantAnalysisScannedRepository,
+): VariantAnalysisScannedRepositoryDto {
+  return {
+    repository: {
+      id: repo.repository.id,
+      fullName: repo.repository.fullName,
+      private: repo.repository.private,
+      stargazersCount: repo.repository.stargazersCount,
+      updatedAt: repo.repository.updatedAt,
+    },
+    analysisStatus: mapVariantAnalysisRepoStatusToDto(repo.analysisStatus),
+    resultCount: repo.resultCount,
+    artifactSizeInBytes: repo.artifactSizeInBytes,
+    failureMessage: repo.failureMessage,
+  };
+}
+
+function mapVariantAnalysisSkippedRepositoriesToDto(
+  repos: VariantAnalysisSkippedRepositories,
+): VariantAnalysisSkippedRepositoriesDto {
+  return {
+    accessMismatchRepos:
+      repos.accessMismatchRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDto(repos.accessMismatchRepos),
+    notFoundRepos:
+      repos.notFoundRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDto(repos.notFoundRepos),
+    noCodeqlDbRepos:
+      repos.noCodeqlDbRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDto(repos.noCodeqlDbRepos),
+    overLimitRepos:
+      repos.overLimitRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDto(repos.overLimitRepos),
+  };
+}
+
+function mapVariantAnalysisSkippedRepositoryGroupToDto(
+  repoGroup: VariantAnalysisSkippedRepositoryGroup,
+): VariantAnalysisSkippedRepositoryGroupDto {
+  return {
+    repositoryCount: repoGroup.repositoryCount,
+    repositories: repoGroup.repositories.map(
+      mapVariantAnalysisSkippedRepositoryToDto,
+    ),
+  };
+}
+
+function mapVariantAnalysisSkippedRepositoryToDto(
+  repo: VariantAnalysisSkippedRepository,
+): VariantAnalysisSkippedRepositoryDto {
+  return {
+    id: repo.id,
+    fullName: repo.fullName,
+    private: repo.private,
+    stargazersCount: repo.stargazersCount,
+    updatedAt: repo.updatedAt,
+  };
+}
+
+function mapVariantAnalysisFailureReasonToDto(
+  failureReason: VariantAnalysisFailureReason,
+): VariantAnalysisFailureReasonDto {
+  switch (failureReason) {
+    case VariantAnalysisFailureReason.NoReposQueried:
+      return VariantAnalysisFailureReasonDto.NoReposQueried;
+    case VariantAnalysisFailureReason.ActionsWorkflowRunFailed:
+      return VariantAnalysisFailureReasonDto.ActionsWorkflowRunFailed;
+    case VariantAnalysisFailureReason.InternalError:
+      return VariantAnalysisFailureReasonDto.InternalError;
+    default:
+      assertNever(failureReason);
+  }
+}
+
+function mapVariantAnalysisRepoStatusToDto(
+  status: VariantAnalysisRepoStatus,
+): VariantAnalysisRepoStatusDto {
+  switch (status) {
+    case VariantAnalysisRepoStatus.Pending:
+      return VariantAnalysisRepoStatusDto.Pending;
+    case VariantAnalysisRepoStatus.InProgress:
+      return VariantAnalysisRepoStatusDto.InProgress;
+    case VariantAnalysisRepoStatus.Succeeded:
+      return VariantAnalysisRepoStatusDto.Succeeded;
+    case VariantAnalysisRepoStatus.Failed:
+      return VariantAnalysisRepoStatusDto.Failed;
+    case VariantAnalysisRepoStatus.Canceled:
+      return VariantAnalysisRepoStatusDto.Canceled;
+    case VariantAnalysisRepoStatus.TimedOut:
+      return VariantAnalysisRepoStatusDto.TimedOut;
+    default:
+      assertNever(status);
+  }
+}
+
+function mapVariantAnalysisStatusToDto(
+  status: VariantAnalysisStatus,
+): VariantAnalysisStatusDto {
+  switch (status) {
+    case VariantAnalysisStatus.InProgress:
+      return VariantAnalysisStatusDto.InProgress;
+    case VariantAnalysisStatus.Succeeded:
+      return VariantAnalysisStatusDto.Succeeded;
+    case VariantAnalysisStatus.Failed:
+      return VariantAnalysisStatusDto.Failed;
+    case VariantAnalysisStatus.Canceled:
+      return VariantAnalysisStatusDto.Canceled;
+    default:
+      assertNever(status);
+  }
+}
+
+function mapQueryLanguageToDto(language: QueryLanguage): QueryLanguageDto {
+  switch (language) {
+    case QueryLanguage.CSharp:
+      return QueryLanguageDto.CSharp;
+    case QueryLanguage.Cpp:
+      return QueryLanguageDto.Cpp;
+    case QueryLanguage.Go:
+      return QueryLanguageDto.Go;
+    case QueryLanguage.Java:
+      return QueryLanguageDto.Java;
+    case QueryLanguage.Javascript:
+      return QueryLanguageDto.Javascript;
+    case QueryLanguage.Python:
+      return QueryLanguageDto.Python;
+    case QueryLanguage.Ruby:
+      return QueryLanguageDto.Ruby;
+    case QueryLanguage.Swift:
+      return QueryLanguageDto.Swift;
+    default:
+      assertNever(language);
+  }
+}
+
+function mapQueryStatusToDto(status: QueryStatus): QueryStatusDto {
+  switch (status) {
+    case QueryStatus.InProgress:
+      return QueryStatusDto.InProgress;
+    case QueryStatus.Completed:
+      return QueryStatusDto.Completed;
+    case QueryStatus.Failed:
+      return QueryStatusDto.Failed;
+    default:
+      assertNever(status);
+  }
+}
--- a/extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto-mapper.ts
+++ b/extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto-mapper.ts
@@ -0,0 +1,253 @@
+import {
+  QueryHistoryVariantAnalysisDto,
+  QueryLanguageDto,
+  QueryStatusDto,
+  VariantAnalysisDto,
+  VariantAnalysisFailureReasonDto,
+  VariantAnalysisRepoStatusDto,
+  VariantAnalysisScannedRepositoryDto,
+  VariantAnalysisSkippedRepositoriesDto,
+  VariantAnalysisSkippedRepositoryDto,
+  VariantAnalysisSkippedRepositoryGroupDto,
+  VariantAnalysisStatusDto,
+} from "./query-history-variant-analysis-dto";
+import {
+  VariantAnalysis,
+  VariantAnalysisFailureReason,
+  VariantAnalysisRepoStatus,
+  VariantAnalysisScannedRepository,
+  VariantAnalysisSkippedRepositories,
+  VariantAnalysisSkippedRepository,
+  VariantAnalysisSkippedRepositoryGroup,
+  VariantAnalysisStatus,
+} from "../../variant-analysis/shared/variant-analysis";
+import { assertNever } from "../../pure/helpers-pure";
+import { QueryLanguage } from "../../common/query-language";
+import { QueryStatus } from "../../query-status";
+import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
+
+export function mapQueryHistoryVariantAnalysisToDomainModel(
+  item: QueryHistoryVariantAnalysisDto,
+): VariantAnalysisHistoryItem {
+  return {
+    t: "variant-analysis",
+    failureReason: item.failureReason,
+    resultCount: item.resultCount,
+    status: mapQueryStatusToDomainModel(item.status),
+    completed: item.completed,
+    variantAnalysis: mapVariantAnalysisToDomainModel(item.variantAnalysis),
+    userSpecifiedLabel: item.userSpecifiedLabel,
+  };
+}
+
+function mapVariantAnalysisToDomainModel(
+  variantAnalysis: VariantAnalysisDto,
+): VariantAnalysis {
+  return {
+    id: variantAnalysis.id,
+    controllerRepo: {
+      id: variantAnalysis.controllerRepo.id,
+      fullName: variantAnalysis.controllerRepo.fullName,
+      private: variantAnalysis.controllerRepo.private,
+    },
+    query: {
+      name: variantAnalysis.query.name,
+      filePath: variantAnalysis.query.filePath,
+      language: mapQueryLanguageToDomainModel(variantAnalysis.query.language),
+      text: variantAnalysis.query.text,
+    },
+    databases: {
+      repositories: variantAnalysis.databases.repositories,
+      repositoryLists: variantAnalysis.databases.repositoryLists,
+      repositoryOwners: variantAnalysis.databases.repositoryOwners,
+    },
+    createdAt: variantAnalysis.createdAt,
+    updatedAt: variantAnalysis.updatedAt,
+    executionStartTime: variantAnalysis.executionStartTime,
+    status: mapVariantAnalysisStatusToDomainModel(variantAnalysis.status),
+    completedAt: variantAnalysis.completedAt,
+    actionsWorkflowRunId: variantAnalysis.actionsWorkflowRunId,
+    failureReason:
+      variantAnalysis.failureReason &&
+      mapVariantAnalysisFailureReasonToDomainModel(
+        variantAnalysis.failureReason,
+      ),
+    scannedRepos:
+      variantAnalysis.scannedRepos &&
+      mapVariantAnalysisScannedRepositoriesToDomainModel(
+        variantAnalysis.scannedRepos,
+      ),
+    skippedRepos:
+      variantAnalysis.skippedRepos &&
+      mapVariantAnalysisSkippedRepositoriesToDomainModel(
+        variantAnalysis.skippedRepos,
+      ),
+  };
+}
+
+function mapVariantAnalysisScannedRepositoriesToDomainModel(
+  repos: VariantAnalysisScannedRepositoryDto[],
+): VariantAnalysisScannedRepository[] {
+  return repos.map(mapVariantAnalysisScannedRepositoryToDomainModel);
+}
+
+function mapVariantAnalysisScannedRepositoryToDomainModel(
+  repo: VariantAnalysisScannedRepositoryDto,
+): VariantAnalysisScannedRepository {
+  return {
+    repository: {
+      id: repo.repository.id,
+      fullName: repo.repository.fullName,
+      private: repo.repository.private,
+      stargazersCount: repo.repository.stargazersCount,
+      updatedAt: repo.repository.updatedAt,
+    },
+    analysisStatus: mapVariantAnalysisRepoStatusToDomainModel(
+      repo.analysisStatus,
+    ),
+    resultCount: repo.resultCount,
+    artifactSizeInBytes: repo.artifactSizeInBytes,
+    failureMessage: repo.failureMessage,
+  };
+}
+
+function mapVariantAnalysisSkippedRepositoriesToDomainModel(
+  repos: VariantAnalysisSkippedRepositoriesDto,
+): VariantAnalysisSkippedRepositories {
+  return {
+    accessMismatchRepos:
+      repos.accessMismatchRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
+        repos.accessMismatchRepos,
+      ),
+    notFoundRepos:
+      repos.notFoundRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
+        repos.notFoundRepos,
+      ),
+    noCodeqlDbRepos:
+      repos.noCodeqlDbRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
+        repos.noCodeqlDbRepos,
+      ),
+    overLimitRepos:
+      repos.overLimitRepos &&
+      mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
+        repos.overLimitRepos,
+      ),
+  };
+}
+
+function mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
+  repoGroup: VariantAnalysisSkippedRepositoryGroupDto,
+): VariantAnalysisSkippedRepositoryGroup {
+  return {
+    repositoryCount: repoGroup.repositoryCount,
+    repositories: repoGroup.repositories.map(
+      mapVariantAnalysisSkippedRepositoryToDomainModel,
+    ),
+  };
+}
+
+function mapVariantAnalysisSkippedRepositoryToDomainModel(
+  repo: VariantAnalysisSkippedRepositoryDto,
+): VariantAnalysisSkippedRepository {
+  return {
+    id: repo.id,
+    fullName: repo.fullName,
+    private: repo.private,
+    stargazersCount: repo.stargazersCount,
+    updatedAt: repo.updatedAt,
+  };
+}
+
+function mapVariantAnalysisFailureReasonToDomainModel(
+  failureReason: VariantAnalysisFailureReasonDto,
+): VariantAnalysisFailureReason {
+  switch (failureReason) {
+    case VariantAnalysisFailureReasonDto.NoReposQueried:
+      return VariantAnalysisFailureReason.NoReposQueried;
+    case VariantAnalysisFailureReasonDto.ActionsWorkflowRunFailed:
+      return VariantAnalysisFailureReason.ActionsWorkflowRunFailed;
+    case VariantAnalysisFailureReasonDto.InternalError:
+      return VariantAnalysisFailureReason.InternalError;
+    default:
+      assertNever(failureReason);
+  }
+}
+
+function mapVariantAnalysisRepoStatusToDomainModel(
+  status: VariantAnalysisRepoStatusDto,
+): VariantAnalysisRepoStatus {
+  switch (status) {
+    case VariantAnalysisRepoStatusDto.Pending:
+      return VariantAnalysisRepoStatus.Pending;
+    case VariantAnalysisRepoStatusDto.InProgress:
+      return VariantAnalysisRepoStatus.InProgress;
+    case VariantAnalysisRepoStatusDto.Succeeded:
+      return VariantAnalysisRepoStatus.Succeeded;
+    case VariantAnalysisRepoStatusDto.Failed:
+      return VariantAnalysisRepoStatus.Failed;
+    case VariantAnalysisRepoStatusDto.Canceled:
+      return VariantAnalysisRepoStatus.Canceled;
+    case VariantAnalysisRepoStatusDto.TimedOut:
+      return VariantAnalysisRepoStatus.TimedOut;
+    default:
+      assertNever(status);
+  }
+}
+
+function mapVariantAnalysisStatusToDomainModel(
+  status: VariantAnalysisStatusDto,
+): VariantAnalysisStatus {
+  switch (status) {
+    case VariantAnalysisStatusDto.InProgress:
+      return VariantAnalysisStatus.InProgress;
+    case VariantAnalysisStatusDto.Succeeded:
+      return VariantAnalysisStatus.Succeeded;
+    case VariantAnalysisStatusDto.Failed:
+      return VariantAnalysisStatus.Failed;
+    case VariantAnalysisStatusDto.Canceled:
+      return VariantAnalysisStatus.Canceled;
+    default:
+      assertNever(status);
+  }
+}
+
+function mapQueryLanguageToDomainModel(
+  language: QueryLanguageDto,
+): QueryLanguage {
+  switch (language) {
+    case QueryLanguageDto.CSharp:
+      return QueryLanguage.CSharp;
+    case QueryLanguageDto.Cpp:
+      return QueryLanguage.Cpp;
+    case QueryLanguageDto.Go:
+      return QueryLanguage.Go;
+    case QueryLanguageDto.Java:
+      return QueryLanguage.Java;
+    case QueryLanguageDto.Javascript:
+      return QueryLanguage.Javascript;
+    case QueryLanguageDto.Python:
+      return QueryLanguage.Python;
+    case QueryLanguageDto.Ruby:
+      return QueryLanguage.Ruby;
+    case QueryLanguageDto.Swift:
+      return QueryLanguage.Swift;
+    default:
+      assertNever(language);
+  }
+}
+
+function mapQueryStatusToDomainModel(status: QueryStatusDto): QueryStatus {
+  switch (status) {
+    case QueryStatusDto.InProgress:
+      return QueryStatus.InProgress;
+    case QueryStatusDto.Completed:
+      return QueryStatus.Completed;
+    case QueryStatusDto.Failed:
+      return QueryStatus.Failed;
+    default:
+      assertNever(status);
+  }
+}
--- a/extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto.ts
+++ b/extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto.ts
@@ -1,27 +1,17 @@
 // Contains models and consts for the data we want to store in the query history store.
 // Changes to these models should be done carefully and account for backwards compatibility of data.

-import { QueryLanguage } from "../../common/query-language";
-import { QueryStatus } from "../../query-status";
-import {
-  VariantAnalysisFailureReason,
-  VariantAnalysisRepoStatus,
-  VariantAnalysisStatus,
-} from "../../variant-analysis/shared/variant-analysis";
-
-// All data points are modelled, except enums.
-
 export interface QueryHistoryVariantAnalysisDto {
  readonly t: "variant-analysis";
  failureReason?: string;
  resultCount?: number;
-  status: QueryStatus;
+  status: QueryStatusDto;
  completed: boolean;
-  variantAnalysis: VariantAnalysisQueryHistoryDto;
+  variantAnalysis: VariantAnalysisDto;
  userSpecifiedLabel?: string;
 }

-export interface VariantAnalysisQueryHistoryDto {
+export interface VariantAnalysisDto {
  id: number;
  controllerRepo: {
    id: number;
@@ -31,7 +21,7 @@ export interface VariantAnalysisQueryHistoryDto {
  query: {
    name: string;
    filePath: string;
-    language: QueryLanguage;
+    language: QueryLanguageDto;
    text: string;
  };
  databases: {
@@ -42,10 +32,10 @@ export interface VariantAnalysisQueryHistoryDto {
  createdAt: string;
  updatedAt: string;
  executionStartTime: number;
-  status: VariantAnalysisStatus;
+  status: VariantAnalysisStatusDto;
  completedAt?: string;
  actionsWorkflowRunId?: number;
-  failureReason?: VariantAnalysisFailureReason;
+  failureReason?: VariantAnalysisFailureReasonDto;
  scannedRepos?: VariantAnalysisScannedRepositoryDto[];
  skippedRepos?: VariantAnalysisSkippedRepositoriesDto;
 }
@@ -58,7 +48,7 @@ export interface VariantAnalysisScannedRepositoryDto {
    stargazersCount: number;
    updatedAt: string | null;
  };
-  analysisStatus: VariantAnalysisRepoStatus;
+  analysisStatus: VariantAnalysisRepoStatusDto;
  resultCount?: number;
  artifactSizeInBytes?: number;
  failureMessage?: string;
@@ -83,3 +73,42 @@ export interface VariantAnalysisSkippedRepositoryDto {
  stargazersCount?: number;
  updatedAt?: string | null;
 }
+
+export enum VariantAnalysisFailureReasonDto {
+  NoReposQueried = "noReposQueried",
+  ActionsWorkflowRunFailed = "actionsWorkflowRunFailed",
+  InternalError = "internalError",
+}
+
+export enum VariantAnalysisRepoStatusDto {
+  Pending = "pending",
+  InProgress = "inProgress",
+  Succeeded = "succeeded",
+  Failed = "failed",
+  Canceled = "canceled",
+  TimedOut = "timedOut",
+}
+
+export enum VariantAnalysisStatusDto {
+  InProgress = "inProgress",
+  Succeeded = "succeeded",
+  Failed = "failed",
+  Canceled = "canceled",
+}
+
+export enum QueryLanguageDto {
+  CSharp = "csharp",
+  Cpp = "cpp",
+  Go = "go",
+  Java = "java",
+  Javascript = "javascript",
+  Python = "python",
+  Ruby = "ruby",
+  Swift = "swift",
+}
+
+export enum QueryStatusDto {
+  InProgress = "InProgress",
+  Completed = "Completed",
+  Failed = "Failed",
+}
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
@@ -27,7 +27,7 @@ import { QueryResultType } from "../../../src/pure/new-messages";
 import { createVSCodeCommandManager } from "../../../src/common/vscode/commands";
 import { AllCommands, QueryServerCommands } from "../../../src/common/commands";

-jest.setTimeout(20_000);
+jest.setTimeout(60_000);

 /**
 * Integration tests for queries
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
@@ -8,8 +8,10 @@ import { DatabaseKind } from "../../../../src/local-databases";
 import * as queryResolver from "../../../../src/contextual/queryResolver";
 import { file } from "tmp-promise";
 import { QueryResultType } from "../../../../src/pure/new-messages";
-import { readFile } from "fs-extra";
+import { readdir, readFile } from "fs-extra";
 import { load } from "js-yaml";
+import { dirname, join } from "path";
+import { fetchExternalApisQuery } from "../../../../src/data-extensions-editor/queries/java";
 import * as helpers from "../../../../src/helpers";
 import { RedactableError } from "../../../../src/pure/errors";

@@ -41,11 +43,6 @@ describe("runQuery", () => {
        resolveQlpacks: jest.fn().mockResolvedValue({
          "my/java-extensions": "/a/b/c/",
        }),
-        resolveQueriesInSuite: jest
-          .fn()
-          .mockResolvedValue([
-            "/home/github/codeql/java/ql/src/Telemetry/FetchExternalAPIs.ql",
-          ]),
      },
      queryRunner: {
        createQueryRun: jest.fn().mockReturnValue({
@@ -58,7 +55,6 @@ describe("runQuery", () => {
        }),
        logger: createMockLogger(),
      },
-      logger: createMockLogger(),
      databaseItem: {
        databaseUri: createMockUri("/a/b/c/src.zip"),
        contents: {
@@ -79,37 +75,12 @@ describe("runQuery", () => {

    expect(result?.resultType).toEqual(QueryResultType.SUCCESS);

-    expect(options.cliServer.resolveQueriesInSuite).toHaveBeenCalledWith(
-      expect.anything(),
-      [],
-    );
-    const suiteFile = options.cliServer.resolveQueriesInSuite.mock.calls[0][0];
-    const suiteFileContents = await readFile(suiteFile, "utf8");
-    const suiteYaml = load(suiteFileContents);
-    expect(suiteYaml).toEqual([
-      {
-        from: "codeql/java-all",
-        queries: ".",
-        include: {
-          id: "java/telemetry/fetch-external-apis",
-        },
-      },
-      {
-        from: "codeql/java-queries",
-        queries: ".",
-        include: {
-          id: "java/telemetry/fetch-external-apis",
-        },
-      },
-    ]);
-
    expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
    expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
    expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(
      "/a/b/c/src.zip",
      {
-        queryPath:
-          "/home/github/codeql/java/ql/src/Telemetry/FetchExternalAPIs.ql",
+        queryPath: expect.stringMatching(/FetchExternalApis\.ql/),
        quickEvalPosition: undefined,
      },
      false,
@@ -119,6 +90,40 @@ describe("runQuery", () => {
      undefined,
      undefined,
    );
+
+    const queryPath =
+      options.queryRunner.createQueryRun.mock.calls[0][1].queryPath;
+    const queryDirectory = dirname(queryPath);
+
+    const queryFiles = await readdir(queryDirectory);
+    expect(queryFiles.sort()).toEqual(
+      ["codeql-pack.yml", "FetchExternalApis.ql", "ExternalApi.qll"].sort(),
+    );
+
+    const suiteFileContents = await readFile(
+      join(queryDirectory, "codeql-pack.yml"),
+      "utf8",
+    );
+    const suiteYaml = load(suiteFileContents);
+    expect(suiteYaml).toEqual({
+      name: "codeql/external-api-usage",
+      version: "0.0.0",
+      dependencies: {
+        "codeql/java-all": "*",
+      },
+    });
+
+    expect(
+      await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"),
+    ).toEqual(fetchExternalApisQuery.mainQuery);
+
+    for (const [filename, contents] of Object.entries(
+      fetchExternalApisQuery.dependencies ?? {},
+    )) {
+      expect(await readFile(join(queryDirectory, filename), "utf8")).toEqual(
+        contents,
+      );
+    }
  });
 });

--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data/variant-analysis/workspace-query-history.json
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data/variant-analysis/workspace-query-history.json
@@ -7,7 +7,11 @@
      "completed": true,
      "variantAnalysis": {
        "id": 98574321397,
-        "controllerRepoId": 128321,
+        "controllerRepo": {
+          "id": 128321,
+          "fullName": "github/codeql",
+          "private": false
+        },
        "query": {
          "name": "Variant Analysis Integration Test 1",
          "filePath": "PLACEHOLDER/q2.ql",
@@ -30,7 +34,11 @@
      "completed": true,
      "variantAnalysis": {
        "id": 98574321397,
-        "controllerRepoId": 128321,
+        "controllerRepo": {
+          "id": 128321,
+          "fullName": "github/codeql",
+          "private": false
+        },
        "query": {
          "name": "Variant Analysis Integration Test 2",
          "filePath": "PLACEHOLDER/q2.ql",