Merge branch 'main' into yer-a-workspace-query

This commit is contained in:
Elena Tanasoiu
2023-04-14 14:00:28 +01:00
committed by GitHub
72 changed files with 8073 additions and 1430 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -1461,6 +1461,7 @@
"fs-extra": "^11.1.1",
"immutable": "^4.0.0",
"js-yaml": "^4.1.0",
"minimatch": "^9.0.0",
"minimist": "~1.2.6",
"msw": "^1.2.0",
"nanoid": "^3.2.0",
@@ -1497,7 +1498,7 @@
"@storybook/addon-essentials": "^6.5.17-alpha.0",
"@storybook/addon-interactions": "^6.5.17-alpha.0",
"@storybook/addon-links": "^6.5.17-alpha.0",
"@storybook/builder-webpack5": "^6.5.17-alpha.0",
"@storybook/builder-webpack5": "^7.0.4",
"@storybook/manager-webpack5": "^6.5.17-alpha.0",
"@storybook/react": "^6.5.17-alpha.0",
"@storybook/testing-library": "^0.0.13",
@@ -1531,7 +1532,7 @@
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",
"@types/vscode": "^1.59.0",
"@types/vscode": "^1.67.0",
"@types/webpack": "^5.28.0",
"@types/webpack-env": "^1.18.0",
"@types/xml2js": "~0.4.4",
@@ -1555,7 +1556,7 @@
"eslint-plugin-react-hooks": "^4.6.0",
"eslint-plugin-storybook": "^0.6.4",
"file-loader": "^6.2.0",
"glob": "^9.3.2",
"glob": "^10.0.0",
"gulp": "^4.0.2",
"gulp-esbuild": "^0.10.5",
"gulp-replace": "^1.1.3",

View File

@@ -13,16 +13,51 @@ import type {
} from "../variant-analysis/shared/variant-analysis";
// A command function matching the signature that VS Code calls when
// a command on a selection is invoked.
export type SelectionCommandFunction<Item> = (
singleItem: Item,
multiSelect: Item[],
// a command is invoked from the title bar of a TreeView with
// canSelectMany set to true.
//
// It is possible to get any combination of singleItem and multiSelect
// to be undefined. This is because it is possible to click a title bar
// option without interacting with any individual items first, or even
// when there are no items present at all.
// If both singleItem and multiSelect are defined, then singleItem will
// be contained within multiSelect.
export type TreeViewTitleMultiSelectionCommandFunction<Item> = (
singleItem: Item | undefined,
multiSelect: Item[] | undefined,
) => Promise<void>;
// A command function matching the signature that VS Code calls when
// a command on a selection is invoked when canSelectMany is false.
export type SingleSelectionCommandFunction<Item> = (
// a command is invoked from a context menu on a TreeView with
// canSelectMany set to true.
//
// singleItem will always be defined and corresponds to the item that
// was hovered or right-clicked. If precisely one item was selected then
// multiSelect will be undefined. If more than one item is selected then
// multiSelect will contain all selected items, including singleItem.
export type TreeViewContextMultiSelectionCommandFunction<Item> = (
singleItem: Item,
multiSelect: Item[] | undefined,
) => Promise<void>;
// A command function matching the signature that VS Code calls when
// a command is invoked from a context menu on a TreeView with
// canSelectMany set to false.
//
// It is guaranteed that precisely one item will be selected.
export type TreeViewContextSingleSelectionCommandFunction<Item> = (
singleItem: Item,
) => Promise<void>;
// A command function matching the signature that VS Code calls when
// a command is invoked from a context menu on the file explorer.
//
// singleItem corresponds to the item that was right-clicked.
// multiSelect will always been defined and non-empty and contains
// all selected items, including singleItem.
export type ExplorerSelectionCommandFunction<Item> = (
singleItem: Item,
multiSelect: Item[],
) => Promise<void>;
/**
@@ -94,7 +129,7 @@ export type LocalQueryCommands = {
"codeQL.runQueryOnMultipleDatabasesContextEditor": (
uri?: Uri,
) => Promise<void>;
"codeQL.runQueries": SelectionCommandFunction<Uri>;
"codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
"codeQL.quickEval": (uri: Uri) => Promise<void>;
"codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
"codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
@@ -119,28 +154,28 @@ export type QueryHistoryCommands = {
"codeQLQueryHistory.sortByCount": () => Promise<void>;
// Commands in the context menu or in the hover menu
"codeQLQueryHistory.openQueryTitleMenu": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openQueryContextMenu": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.removeHistoryItemTitleMenu": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.removeHistoryItemContextMenu": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.removeHistoryItemContextInline": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.renameItem": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.compareWith": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLog": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLogSummary": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLogViewer": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showQueryLog": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showQueryText": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openQueryDirectory": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.cancel": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.exportResults": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewCsvResults": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewCsvAlerts": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewSarifAlerts": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewDil": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.itemClicked": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openOnGithub": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.copyRepoList": SelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openQueryTitleMenu": TreeViewTitleMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openQueryContextMenu": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.removeHistoryItemTitleMenu": TreeViewTitleMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.removeHistoryItemContextMenu": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showQueryLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showQueryText": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openQueryDirectory": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.cancel": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.exportResults": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewCsvResults": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewCsvAlerts": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewSarifAlerts": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.viewDil": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.itemClicked": TreeViewTitleMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.openOnGithub": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.copyRepoList": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
// Commands in the command palette
"codeQL.exportSelectedVariantAnalysisResults": () => Promise<void>;
@@ -173,11 +208,11 @@ export type LocalDatabasesCommands = {
) => Promise<void>;
// Database panel selection commands
"codeQLDatabases.removeDatabase": SelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.upgradeDatabase": SelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.renameDatabase": SelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.openDatabaseFolder": SelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.addDatabaseSource": SelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.removeDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.upgradeDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.renameDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.openDatabaseFolder": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
"codeQLDatabases.addDatabaseSource": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
// Codespace template commands
"codeQL.setDefaultTourDatabase": () => Promise<void>;
@@ -222,11 +257,11 @@ export type DatabasePanelCommands = {
"codeQLVariantAnalysisRepositories.addNewList": () => Promise<void>;
"codeQLVariantAnalysisRepositories.setupControllerRepository": () => Promise<void>;
"codeQLVariantAnalysisRepositories.setSelectedItem": SingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.setSelectedItemContextMenu": SingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": SingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.renameItemContextMenu": SingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.removeItemContextMenu": SingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.setSelectedItem": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.setSelectedItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
};
export type AstCfgCommands = {

View File

@@ -53,7 +53,7 @@ export class DataExtensionsEditorModule {
return {
"codeQL.openDataExtensionsEditor": async () =>
withProgress(
async (progress) => {
async (progress, token) => {
const db = this.databaseManager.currentDatabaseItem;
if (!db) {
void showAndLogErrorMessage("No database selected");
@@ -69,7 +69,9 @@ export class DataExtensionsEditorModule {
const modelFile = await pickExtensionPackModelFile(
this.cliServer,
db,
progress,
token,
);
if (!modelFile) {
return;

View File

@@ -14,11 +14,11 @@ import {
import { ProgressUpdate } from "../progress";
import { QueryRunner } from "../queryRunner";
import {
showAndLogErrorMessage,
showAndLogExceptionWithTelemetry,
showAndLogWarningMessage,
} from "../helpers";
import { extLogger } from "../common";
import { readFile, writeFile } from "fs-extra";
import { outputFile, readFile } from "fs-extra";
import { load as loadYaml } from "js-yaml";
import { DatabaseItem, DatabaseManager } from "../local-databases";
import { CodeQLCliServer } from "../cli";
@@ -150,7 +150,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
): Promise<void> {
const yaml = createDataExtensionYaml(externalApiUsages, modeledMethods);
await writeFile(this.modelFilename, yaml);
await outputFile(this.modelFilename, yaml);
void extLogger.log(`Saved data extension YAML to ${this.modelFilename}`);
}
@@ -166,7 +166,9 @@ export class DataExtensionsEditorView extends AbstractWebview<
const existingModeledMethods = loadDataExtensionYaml(data);
if (!existingModeledMethods) {
void showAndLogWarningMessage("Failed to parse data extension YAML.");
void showAndLogErrorMessage(
`Failed to parse data extension YAML ${this.modelFilename}.`,
);
return;
}
@@ -175,7 +177,11 @@ export class DataExtensionsEditorView extends AbstractWebview<
modeledMethods: existingModeledMethods,
});
} catch (e: unknown) {
void extLogger.log(`Unable to read data extension YAML: ${e}`);
void showAndLogErrorMessage(
`Unable to read data extension YAML ${
this.modelFilename
}: ${getErrorMessage(e)}`,
);
}
}
@@ -188,7 +194,6 @@ export class DataExtensionsEditorView extends AbstractWebview<
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
queryStorageDir: this.queryStorageDir,
logger: extLogger,
progress: (progressUpdate: ProgressUpdate) => {
void this.showProgress(progressUpdate, 1500);
},
@@ -208,7 +213,6 @@ export class DataExtensionsEditorView extends AbstractWebview<
const bqrsChunk = await readQueryResults({
cliServer: this.cliServer,
bqrsPath: queryResult.outputDir.bqrsPath,
logger: extLogger,
});
if (!bqrsChunk) {
await this.clearProgress();
@@ -233,7 +237,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
void showAndLogExceptionWithTelemetry(
redactableError(
asError(err),
)`Failed to load external APi usages: ${getErrorMessage(err)}`,
)`Failed to load external API usages: ${getErrorMessage(err)}`,
);
}
}

View File

@@ -0,0 +1,45 @@
{
"type": "object",
"properties": {
"extensions": {
"type": "array",
"items": {
"type": "object",
"required": ["addsTo", "data"],
"properties": {
"addsTo": {
"type": "object",
"required": ["pack", "extensible"],
"properties": {
"pack": {
"type": "string"
},
"extensible": {
"type": "string"
}
}
},
"data": {
"type": "array",
"items": {
"type": "array",
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "boolean"
},
{
"type": "number"
}
]
}
}
}
}
}
}
}
}

View File

@@ -1,21 +1,49 @@
import { relative, sep } from "path";
import { window } from "vscode";
import { join, relative, resolve, sep } from "path";
import { outputFile, pathExists, readFile } from "fs-extra";
import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import { minimatch } from "minimatch";
import { CancellationToken, window } from "vscode";
import { CodeQLCliServer } from "../cli";
import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from "../helpers";
import {
getOnDiskWorkspaceFolders,
getOnDiskWorkspaceFoldersObjects,
showAndLogErrorMessage,
} from "../helpers";
import { ProgressCallback } from "../progress";
import { DatabaseItem } from "../local-databases";
import { getQlPackPath, QLPACK_FILENAMES } from "../pure/ql";
const maxStep = 3;
const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
const packNameRegex = new RegExp(
`^(?:(?<scope>${packNamePartRegex.source})/)?(?<name>${packNamePartRegex.source})$`,
);
const packNameLength = 128;
export async function pickExtensionPackModelFile(
cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
databaseItem: Pick<DatabaseItem, "name">,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string | undefined> {
const extensionPackPath = await pickExtensionPack(cliServer, progress);
const extensionPackPath = await pickExtensionPack(
cliServer,
databaseItem,
progress,
token,
);
if (!extensionPackPath) {
return;
}
const modelFile = await pickModelFile(cliServer, progress, extensionPackPath);
const modelFile = await pickModelFile(
cliServer,
databaseItem,
extensionPackPath,
progress,
token,
);
if (!modelFile) {
return;
}
@@ -25,7 +53,9 @@ export async function pickExtensionPackModelFile(
async function pickExtensionPack(
cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
databaseItem: Pick<DatabaseItem, "name">,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string | undefined> {
progress({
message: "Resolving extension packs...",
@@ -36,10 +66,20 @@ async function pickExtensionPack(
// Get all existing extension packs in the workspace
const additionalPacks = getOnDiskWorkspaceFolders();
const extensionPacks = await cliServer.resolveQlpacks(additionalPacks, true);
const options = Object.keys(extensionPacks).map((pack) => ({
label: pack,
extensionPack: pack,
}));
if (Object.keys(extensionPacks).length === 0) {
return pickNewExtensionPack(databaseItem, token);
}
const options: Array<{ label: string; extensionPack: string | null }> =
Object.keys(extensionPacks).map((pack) => ({
label: pack,
extensionPack: pack,
}));
options.push({
label: "Create new extension pack",
extensionPack: null,
});
progress({
message: "Choosing extension pack...",
@@ -47,13 +87,21 @@ async function pickExtensionPack(
maxStep,
});
const extensionPackOption = await window.showQuickPick(options, {
title: "Select extension pack to use",
});
const extensionPackOption = await window.showQuickPick(
options,
{
title: "Select extension pack to use",
},
token,
);
if (!extensionPackOption) {
return undefined;
}
if (!extensionPackOption.extensionPack) {
return pickNewExtensionPack(databaseItem, token);
}
const extensionPackPaths = extensionPacks[extensionPackOption.extensionPack];
if (extensionPackPaths.length !== 1) {
void showAndLogErrorMessage(
@@ -74,8 +122,10 @@ async function pickExtensionPack(
async function pickModelFile(
cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
progress: ProgressCallback,
databaseItem: Pick<DatabaseItem, "name">,
extensionPackPath: string,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string | undefined> {
// Find the existing model files in the extension pack
const additionalPacks = getOnDiskWorkspaceFolders();
@@ -92,13 +142,21 @@ async function pickModelFile(
}
}
const fileOptions: Array<{ label: string; file: string }> = [];
if (modelFiles.size === 0) {
return pickNewModelFile(databaseItem, extensionPackPath, token);
}
const fileOptions: Array<{ label: string; file: string | null }> = [];
for (const file of modelFiles) {
fileOptions.push({
label: relative(extensionPackPath, file).replaceAll(sep, "/"),
file,
});
}
fileOptions.push({
label: "Create new model file",
file: null,
});
progress({
message: "Choosing model file...",
@@ -106,13 +164,186 @@ async function pickModelFile(
maxStep,
});
const fileOption = await window.showQuickPick(fileOptions, {
title: "Select model file to use",
});
const fileOption = await window.showQuickPick(
fileOptions,
{
title: "Select model file to use",
},
token,
);
if (!fileOption) {
return undefined;
}
if (fileOption.file) {
return fileOption.file;
}
return pickNewModelFile(databaseItem, extensionPackPath, token);
}
async function pickNewExtensionPack(
databaseItem: Pick<DatabaseItem, "name">,
token: CancellationToken,
): Promise<string | undefined> {
const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
const workspaceFolderOptions = workspaceFolders.map((folder) => ({
label: folder.name,
detail: folder.uri.fsPath,
path: folder.uri.fsPath,
}));
// We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
// we only want to include on-disk workspace folders.
const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
title: "Select workspace folder to create extension pack in",
});
if (!workspaceFolder) {
return undefined;
}
const packName = await window.showInputBox(
{
title: "Create new extension pack",
prompt: "Enter name of extension pack",
placeHolder: `e.g. ${databaseItem.name}-extensions`,
validateInput: async (value: string): Promise<string | undefined> => {
if (!value) {
return "Pack name must not be empty";
}
if (value.length > packNameLength) {
return `Pack name must be no longer than ${packNameLength} characters`;
}
const matches = packNameRegex.exec(value);
if (!matches?.groups) {
return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
}
const packPath = join(workspaceFolder.path, matches.groups.name);
if (await pathExists(packPath)) {
return `A pack already exists at ${packPath}`;
}
return undefined;
},
},
token,
);
if (!packName) {
return undefined;
}
const matches = packNameRegex.exec(packName);
if (!matches?.groups) {
return;
}
return fileOption.file;
const name = matches.groups.name;
const packPath = join(workspaceFolder.path, name);
if (await pathExists(packPath)) {
return undefined;
}
const packYamlPath = join(packPath, "codeql-pack.yml");
await outputFile(
packYamlPath,
dumpYaml({
name,
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
}),
);
return packPath;
}
async function pickNewModelFile(
databaseItem: Pick<DatabaseItem, "name">,
extensionPackPath: string,
token: CancellationToken,
) {
const qlpackPath = await getQlPackPath(extensionPackPath);
if (!qlpackPath) {
void showAndLogErrorMessage(
`Could not find any of ${QLPACK_FILENAMES.join(
", ",
)} in ${extensionPackPath}`,
);
return undefined;
}
const qlpack = await loadYaml(await readFile(qlpackPath, "utf8"), {
filename: qlpackPath,
});
if (typeof qlpack !== "object" || qlpack === null) {
void showAndLogErrorMessage(`Could not parse ${qlpackPath}`);
return undefined;
}
const dataExtensionPatternsValue = qlpack.dataExtensions;
if (
!(
Array.isArray(dataExtensionPatternsValue) ||
typeof dataExtensionPatternsValue === "string"
)
) {
void showAndLogErrorMessage(
`Expected 'dataExtensions' to be a string or an array in ${qlpackPath}`,
);
return undefined;
}
// The YAML allows either a string or an array of strings
const dataExtensionPatterns = Array.isArray(dataExtensionPatternsValue)
? dataExtensionPatternsValue
: [dataExtensionPatternsValue];
const filename = await window.showInputBox(
{
title: "Enter the name of the new model file",
value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
validateInput: async (value: string): Promise<string | undefined> => {
if (value === "") {
return "File name must not be empty";
}
const path = resolve(extensionPackPath, value);
if (await pathExists(path)) {
return "File already exists";
}
const notInExtensionPack = relative(extensionPackPath, path).startsWith(
"..",
);
if (notInExtensionPack) {
return "File must be in the extension pack";
}
const matchesPattern = dataExtensionPatterns.some((pattern) =>
minimatch(value, pattern, { matchBase: true }),
);
if (!matchesPattern) {
return `File must match one of the patterns in 'dataExtensions' in ${qlpackPath}`;
}
return undefined;
},
},
token,
);
if (!filename) {
return undefined;
}
return resolve(extensionPackPath, filename);
}

View File

@@ -1,21 +1,27 @@
import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
import { qlpackOfDatabase } from "../contextual/queryResolver";
import { file } from "tmp-promise";
import { dir } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump as dumpYaml } from "js-yaml";
import { getOnDiskWorkspaceFolders } from "../helpers";
import { Logger, TeeLogger } from "../common";
import {
getOnDiskWorkspaceFolders,
showAndLogExceptionWithTelemetry,
} from "../helpers";
import { TeeLogger } from "../common";
import { CancellationToken } from "vscode";
import { CodeQLCliServer } from "../cli";
import { DatabaseItem } from "../local-databases";
import { ProgressCallback } from "../progress";
import { fetchExternalApiQueries } from "./queries";
import { QueryResultType } from "../pure/new-messages";
import { join } from "path";
import { redactableError } from "../pure/errors";
import { QueryLanguage } from "../common/query-language";
export type RunQueryOptions = {
cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveQueriesInSuite">;
cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
databaseItem: Pick<DatabaseItem, "contents" | "databaseUri" | "language">;
queryStorageDir: string;
logger: Logger;
progress: ProgressCallback;
token: CancellationToken;
@@ -26,54 +32,53 @@ export async function runQuery({
queryRunner,
databaseItem,
queryStorageDir,
logger,
progress,
token,
}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
const qlpacks = await qlpackOfDatabase(cliServer, databaseItem);
// The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
// move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
// This is intentionally not pretty code, as it will be removed soon.
// For a reference of what this should do in the future, see the previous implementation in
// https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
const packsToSearch = [qlpacks.dbschemePack];
if (qlpacks.queryPack) {
packsToSearch.push(qlpacks.queryPack);
const query = fetchExternalApiQueries[databaseItem.language as QueryLanguage];
if (!query) {
void showAndLogExceptionWithTelemetry(
redactableError`No external API usage query found for language ${databaseItem.language}`,
);
return;
}
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
for (const qlpack of packsToSearch) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: {
id: `${databaseItem.language}/telemetry/fetch-external-apis`,
},
});
const queryDir = (await dir({ unsafeCleanup: true })).path;
const queryFile = join(queryDir, "FetchExternalApis.ql");
await writeFile(queryFile, query.mainQuery, "utf8");
if (query.dependencies) {
for (const [filename, contents] of Object.entries(query.dependencies)) {
const dependencyFile = join(queryDir, filename);
await writeFile(dependencyFile, contents, "utf8");
}
}
await writeFile(suiteFile, dumpYaml(suiteYaml), "utf8");
const syntheticQueryPack = {
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${databaseItem.language}-all`]: "*",
},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8");
const additionalPacks = getOnDiskWorkspaceFolders();
const extensionPacks = Object.keys(
await cliServer.resolveQlpacks(additionalPacks, true),
);
const queries = await cliServer.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
if (queries.length !== 1) {
void logger.log(`Expected exactly one query, got ${queries.length}`);
return;
}
const query = queries[0];
const queryRun = queryRunner.createQueryRun(
databaseItem.databaseUri.fsPath,
{ queryPath: query, quickEvalPosition: undefined },
{ queryPath: queryFile, quickEvalPosition: undefined },
false,
getOnDiskWorkspaceFolders(),
extensionPacks,
@@ -82,28 +87,37 @@ export async function runQuery({
undefined,
);
return queryRun.evaluate(
const completedQuery = await queryRun.evaluate(
progress,
token,
new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
);
if (completedQuery.resultType !== QueryResultType.SUCCESS) {
void showAndLogExceptionWithTelemetry(
redactableError`External API usage query failed: ${
completedQuery.message ?? "No message"
}`,
);
return;
}
return completedQuery;
}
export type GetResultsOptions = {
cliServer: Pick<CodeQLCliServer, "bqrsInfo" | "bqrsDecode">;
bqrsPath: string;
logger: Logger;
};
export async function readQueryResults({
cliServer,
bqrsPath,
logger,
}: GetResultsOptions) {
const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
if (bqrsInfo["result-sets"].length !== 1) {
void logger.log(
`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
void showAndLogExceptionWithTelemetry(
redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
);
return undefined;
}

View File

@@ -4,13 +4,18 @@ import { join } from "path";
import { QueryRunner } from "../queryRunner";
import { CodeQLCliServer } from "../cli";
import { TeeLogger } from "../common";
import { extensiblePredicateDefinitions } from "./yaml";
import { extensiblePredicateDefinitions } from "./predicates";
import { ProgressCallback } from "../progress";
import { getOnDiskWorkspaceFolders } from "../helpers";
import {
getOnDiskWorkspaceFolders,
showAndLogExceptionWithTelemetry,
} from "../helpers";
import {
ModeledMethodType,
ModeledMethodWithSignature,
} from "./modeled-method";
import { redactableError } from "../pure/errors";
import { QueryResultType } from "../pure/new-messages";
type FlowModelOptions = {
cliServer: CodeQLCliServer;
@@ -67,13 +72,21 @@ async function getModeledMethodsFromFlow(
token,
new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
);
if (queryResult.resultType !== QueryResultType.SUCCESS) {
void showAndLogExceptionWithTelemetry(
redactableError`Failed to run ${queryName} query: ${
queryResult.message ?? "No message"
}`,
);
return [];
}
const bqrsPath = queryResult.outputDir.bqrsPath;
const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
if (bqrsInfo["result-sets"].length !== 1) {
throw new Error(
`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
void showAndLogExceptionWithTelemetry(
redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length} for ${queryName}`,
);
}

View File

@@ -0,0 +1,138 @@
import { ExternalApiUsage } from "./external-api-usage";
import {
ModeledMethod,
ModeledMethodType,
ModeledMethodWithSignature,
} from "./modeled-method";
export type ExternalApiUsageByType = {
externalApiUsage: ExternalApiUsage;
modeledMethod: ModeledMethod;
};
export type ExtensiblePredicateDefinition = {
extensiblePredicate: string;
generateMethodDefinition: (method: ExternalApiUsageByType) => Tuple[];
readModeledMethod: (row: Tuple[]) => ModeledMethodWithSignature;
supportedKinds?: string[];
};
type Tuple = boolean | number | string;
function readRowToMethod(row: Tuple[]): string {
return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
}
export const extensiblePredicateDefinitions: Record<
Exclude<ModeledMethodType, "none">,
ExtensiblePredicateDefinition
> = {
source: {
extensiblePredicate: "sourceModel",
// extensible predicate sourceModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"",
method.modeledMethod.output,
method.modeledMethod.kind,
"manual",
],
readModeledMethod: (row) => ({
signature: readRowToMethod(row),
modeledMethod: {
type: "source",
input: "",
output: row[6] as string,
kind: row[7] as string,
},
}),
supportedKinds: ["remote"],
},
sink: {
extensiblePredicate: "sinkModel",
// extensible predicate sinkModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"",
method.modeledMethod.input,
method.modeledMethod.kind,
"manual",
],
readModeledMethod: (row) => ({
signature: readRowToMethod(row),
modeledMethod: {
type: "sink",
input: row[6] as string,
output: "",
kind: row[7] as string,
},
}),
supportedKinds: ["sql", "xss", "logging"],
},
summary: {
extensiblePredicate: "summaryModel",
// extensible predicate summaryModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"",
method.modeledMethod.input,
method.modeledMethod.output,
method.modeledMethod.kind,
"manual",
],
readModeledMethod: (row) => ({
signature: readRowToMethod(row),
modeledMethod: {
type: "summary",
input: row[6] as string,
output: row[7] as string,
kind: row[8] as string,
},
}),
supportedKinds: ["taint", "value"],
},
neutral: {
extensiblePredicate: "neutralModel",
// extensible predicate neutralModel(
// string package, string type, string name, string signature, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"manual",
],
readModeledMethod: (row) => ({
signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
modeledMethod: {
type: "neutral",
input: "",
output: "",
kind: "",
},
}),
},
};

View File

@@ -0,0 +1,7 @@
import { fetchExternalApisQuery as javaFetchExternalApisQuery } from "./java";
import { Query } from "./query";
import { QueryLanguage } from "../../common/query-language";
export const fetchExternalApiQueries: Partial<Record<QueryLanguage, Query>> = {
[QueryLanguage.Java]: javaFetchExternalApisQuery,
};

View File

@@ -0,0 +1,183 @@
import { Query } from "./query";
export const fetchExternalApisQuery: Query = {
mainQuery: `/**
* @name Usage of APIs coming from external libraries
* @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
* @tags telemetry
* @id java/telemetry/fetch-external-apis
*/
import java
import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
import ExternalApi
private Call aUsage(ExternalApi api) {
result.getCallee().getSourceDeclaration() = api and
not result.getFile() instanceof GeneratedFile
}
private boolean isSupported(ExternalApi api) {
api.isSupported() and result = true
or
api = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() and result = true
or
not api.isSupported() and
not api = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() and
result = false
}
from ExternalApi api, string apiName, boolean supported, Call usage
where
apiName = api.getApiName() and
supported = isSupported(api) and
usage = aUsage(api)
select apiName, supported, usage
`,
dependencies: {
"ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
private import java
private import semmle.code.java.dataflow.DataFlow
private import semmle.code.java.dataflow.ExternalFlow
private import semmle.code.java.dataflow.FlowSources
private import semmle.code.java.dataflow.FlowSummary
private import semmle.code.java.dataflow.internal.DataFlowPrivate
private import semmle.code.java.dataflow.TaintTracking
pragma[nomagic]
private predicate isTestPackage(Package p) {
p.getName()
.matches([
"org.junit%", "junit.%", "org.mockito%", "org.assertj%",
"com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
"org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
"org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
"org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
"org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
"org.testng%"
])
}
/**
* A test library.
*/
private class TestLibrary extends RefType {
TestLibrary() { isTestPackage(this.getPackage()) }
}
private string containerAsJar(Container container) {
if container instanceof JarFile then result = container.getBaseName() else result = "rt.jar"
}
/** Holds if the given callable is not worth supporting. */
private predicate isUninteresting(Callable c) {
c.getDeclaringType() instanceof TestLibrary or
c.(Constructor).isParameterless()
}
/**
* An external API from either the Standard Library or a 3rd party library.
*/
class ExternalApi extends Callable {
ExternalApi() { not this.fromSource() and not isUninteresting(this) }
/**
* Gets information about the external API in the form expected by the MaD modeling framework.
*/
string getApiName() {
result =
this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
"#" + this.getName() + paramsString(this)
}
/**
* Gets the jar file containing this API. Normalizes the Java Runtime to "rt.jar" despite the presence of modules.
*/
string jarContainer() { result = containerAsJar(this.getCompilationUnit().getParentContainer*()) }
/** Gets a node that is an input to a call to this API. */
private DataFlow::Node getAnInput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr().(Argument).getCall() = call or
result.(ArgumentNode).getCall().asCall() = call
)
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr() = call or
result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
)
}
/** Holds if this API has a supported summary. */
pragma[nomagic]
predicate hasSummary() {
this = any(SummarizedCallable sc).asCallable() or
TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
}
pragma[nomagic]
predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
}
/** Holds if this API is a known sink. */
pragma[nomagic]
predicate isSink() { sinkNode(this.getAnInput(), _) }
/** Holds if this API is supported by existing CodeQL libraries, that is, it is either a recognized source or sink or has a flow summary. */
predicate isSupported() { this.hasSummary() or this.isSource() or this.isSink() }
}
/** DEPRECATED: Alias for ExternalApi */
deprecated class ExternalAPI = ExternalApi;
/**
* Gets the limit for the number of results produced by a telemetry query.
*/
int resultLimit() { result = 1000 }
/**
* Holds if it is relevant to count usages of \`api\`.
*/
signature predicate relevantApi(ExternalApi api);
/**
* Given a predicate to count relevant API usages, this module provides a predicate
* for restricting the number or returned results based on a certain limit.
*/
module Results<relevantApi/1 getRelevantUsages> {
private int getUsages(string apiName) {
result =
strictcount(Call c, ExternalApi api |
c.getCallee().getSourceDeclaration() = api and
not c.getFile() instanceof GeneratedFile and
apiName = api.getApiName() and
getRelevantUsages(api)
)
}
private int getOrder(string apiInfo) {
apiInfo =
rank[result](string info, int usages |
usages = getUsages(info)
|
info order by usages desc, info
)
}
/**
* Holds if there exists an API with \`apiName\` that is being used \`usages\` times
* and if it is in the top results (guarded by resultLimit).
*/
predicate restrict(string apiName, int usages) {
usages = getUsages(apiName) and
getOrder(apiName) <= resultLimit()
}
}
`,
},
};

View File

@@ -0,0 +1,6 @@
export type Query = {
mainQuery: string;
dependencies?: {
[filename: string]: string;
};
};

View File

@@ -1,9 +1,17 @@
import Ajv from "ajv";
import { ExternalApiUsage } from "./external-api-usage";
import {
ModeledMethod,
ModeledMethodType,
ModeledMethodWithSignature,
} from "./modeled-method";
import { extensiblePredicateDefinitions } from "./predicates";
import * as dataSchemaJson from "./data-schema.json";
const ajv = new Ajv({ allErrors: true });
const dataSchemaValidate = ajv.compile(dataSchemaJson);
type ExternalApiUsageByType = {
externalApiUsage: ExternalApiUsage;
@@ -16,120 +24,6 @@ type ExtensiblePredicateDefinition = {
readModeledMethod: (row: any[]) => ModeledMethodWithSignature;
};
function readRowToMethod(row: any[]): string {
return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
}
export const extensiblePredicateDefinitions: Record<
Exclude<ModeledMethodType, "none">,
ExtensiblePredicateDefinition
> = {
source: {
extensiblePredicate: "sourceModel",
// extensible predicate sourceModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"",
method.modeledMethod.output,
method.modeledMethod.kind,
"manual",
],
readModeledMethod: (row) => ({
signature: readRowToMethod(row),
modeledMethod: {
type: "source",
input: "",
output: row[6],
kind: row[7],
},
}),
},
sink: {
extensiblePredicate: "sinkModel",
// extensible predicate sinkModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"",
method.modeledMethod.input,
method.modeledMethod.kind,
"manual",
],
readModeledMethod: (row) => ({
signature: readRowToMethod(row),
modeledMethod: {
type: "sink",
input: row[6],
output: "",
kind: row[7],
},
}),
},
summary: {
extensiblePredicate: "summaryModel",
// extensible predicate summaryModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"",
method.modeledMethod.input,
method.modeledMethod.output,
method.modeledMethod.kind,
"manual",
],
readModeledMethod: (row) => ({
signature: readRowToMethod(row),
modeledMethod: {
type: "summary",
input: row[6],
output: row[7],
kind: row[8],
},
}),
},
neutral: {
extensiblePredicate: "neutralModel",
// extensible predicate neutralModel(
// string package, string type, string name, string signature, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
"manual",
],
readModeledMethod: (row) => ({
signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
modeledMethod: {
type: "neutral",
input: "",
output: "",
kind: "",
},
}),
},
};
function createDataProperty(
methods: ExternalApiUsageByType[],
definition: ExtensiblePredicateDefinition,
@@ -191,8 +85,14 @@ ${extensions.join("\n")}`;
export function loadDataExtensionYaml(
data: any,
): Record<string, ModeledMethod> | undefined {
if (typeof data !== "object") {
return undefined;
dataSchemaValidate(data);
if (dataSchemaValidate.errors) {
throw new Error(
`Invalid data extension YAML: ${dataSchemaValidate.errors
.map((error) => `${error.instancePath} ${error.message}`)
.join(", ")}`,
);
}
const extensions = data.extensions;
@@ -204,19 +104,8 @@ export function loadDataExtensionYaml(
for (const extension of extensions) {
const addsTo = extension.addsTo;
if (typeof addsTo !== "object") {
continue;
}
const extensible = addsTo.extensible;
if (typeof extensible !== "string") {
continue;
}
const data = extension.data;
if (!Array.isArray(data)) {
continue;
}
const definition = Object.values(extensiblePredicateDefinitions).find(
(definition) => definition.extensiblePredicate === extensible,

View File

@@ -34,11 +34,11 @@ import { DatabasePanelCommands } from "../../common/commands";
import { App } from "../../common/app";
export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
kind: string;
remoteDatabaseKind: string;
}
export interface AddListQuickPickItem extends QuickPickItem {
kind: DbListKind;
databaseKind: DbListKind;
}
export class DbPanel extends DisposableObject {
@@ -113,19 +113,19 @@ export class DbPanel extends DisposableObject {
) {
await this.addNewRemoteRepo(highlightedItem.parentListName);
} else {
const quickPickItems = [
const quickPickItems: RemoteDatabaseQuickPickItem[] = [
{
label: "$(repo) From a GitHub repository",
detail: "Add a variant analysis repository from GitHub",
alwaysShow: true,
kind: "repo",
remoteDatabaseKind: "repo",
},
{
label: "$(organization) All repositories of a GitHub org or owner",
detail:
"Add a variant analysis list of repositories from a GitHub organization/owner",
alwaysShow: true,
kind: "owner",
remoteDatabaseKind: "owner",
},
];
const databaseKind =
@@ -142,9 +142,9 @@ export class DbPanel extends DisposableObject {
// We set 'true' to make this a silent exception.
throw new UserCancellationException("No repository selected", true);
}
if (databaseKind.kind === "repo") {
if (databaseKind.remoteDatabaseKind === "repo") {
await this.addNewRemoteRepo();
} else if (databaseKind.kind === "owner") {
} else if (databaseKind.remoteDatabaseKind === "owner") {
await this.addNewRemoteOwner();
}
}

View File

@@ -16,6 +16,7 @@ import {
window as Window,
workspace,
env,
WorkspaceFolder,
} from "vscode";
import { CodeQLCliServer, QlpacksInfo } from "./cli";
import { UserCancellationException } from "./progress";
@@ -249,16 +250,21 @@ export async function showInformationMessageWithAction(
}
/** Gets all active workspace folders that are on the filesystem. */
export function getOnDiskWorkspaceFolders() {
export function getOnDiskWorkspaceFoldersObjects() {
const workspaceFolders = workspace.workspaceFolders || [];
const diskWorkspaceFolders: string[] = [];
const diskWorkspaceFolders: WorkspaceFolder[] = [];
for (const workspaceFolder of workspaceFolders) {
if (workspaceFolder.uri.scheme === "file")
diskWorkspaceFolders.push(workspaceFolder.uri.fsPath);
diskWorkspaceFolders.push(workspaceFolder);
}
return diskWorkspaceFolders;
}
/** Gets all active workspace folders that are on the filesystem. */
export function getOnDiskWorkspaceFolders() {
return getOnDiskWorkspaceFoldersObjects().map((folder) => folder.uri.fsPath);
}
/** Check if folder is already present in workspace */
export function isFolderAlreadyInWorkspace(folderName: string) {
const workspaceFolders = workspace.workspaceFolders || [];

View File

@@ -267,7 +267,7 @@ export class LocalQueries extends DisposableObject {
);
}
private async runQueries(_: Uri | undefined, multi: Uri[]): Promise<void> {
private async runQueries(_: unknown, multi: Uri[]): Promise<void> {
await withProgress(
async (progress, token) => {
const maxQueryCount = MAX_QUERIES.getValue() as number;

View File

@@ -39,10 +39,7 @@ import {
QueryStatus,
variantAnalysisStatusToQueryStatus,
} from "../query-status";
import {
readQueryHistoryFromFile,
writeQueryHistoryToFile,
} from "./store/query-history-store";
import { readQueryHistoryFromFile, writeQueryHistoryToFile } from "./store";
import { pathExists } from "fs-extra";
import { CliVersionConstraint } from "../cli";
import { HistoryItemLabelProvider } from "./history-item-label-provider";
@@ -402,8 +399,8 @@ export class QueryHistoryManager extends DisposableObject {
}
async handleOpenQuery(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
singleItem: QueryHistoryInfo | undefined,
multiSelect: QueryHistoryInfo[] | undefined,
): Promise<void> {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -465,8 +462,8 @@ export class QueryHistoryManager extends DisposableObject {
}
async handleRemoveHistoryItem(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[] = [],
singleItem: QueryHistoryInfo | undefined,
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -566,14 +563,14 @@ export class QueryHistoryManager extends DisposableObject {
async handleRenameItem(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
): Promise<void> {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
multiSelect,
);
if (!this.assertSingleQuery(finalMultiSelect)) {
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem) {
return;
}
@@ -595,7 +592,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleCompareWith(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -633,8 +630,8 @@ export class QueryHistoryManager extends DisposableObject {
}
async handleItemClicked(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[] = [],
singleItem: QueryHistoryInfo | undefined,
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -668,7 +665,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleShowQueryLog(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
// Local queries only
if (!this.assertSingleQuery(multiSelect) || singleItem?.t !== "local") {
@@ -709,7 +706,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleOpenQueryDirectory(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -783,7 +780,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleShowEvalLog(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -811,7 +808,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleShowEvalLogSummary(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -849,7 +846,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleShowEvalLogViewer(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -889,7 +886,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleCancel(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -954,7 +951,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleViewSarifAlerts(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -988,7 +985,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleViewCsvResults(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -1016,7 +1013,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleViewCsvAlerts(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -1044,7 +1041,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleViewDil(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -1071,7 +1068,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleOpenOnGithub(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -1096,7 +1093,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleCopyRepoList(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -1120,7 +1117,7 @@ export class QueryHistoryManager extends DisposableObject {
async handleExportResults(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
multiSelect: QueryHistoryInfo[] | undefined,
): Promise<void> {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(
singleItem,
@@ -1295,10 +1292,10 @@ export class QueryHistoryManager extends DisposableObject {
* @param multiSelect a multi-select or undefined if no items are selected
*/
private determineSelection(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
singleItem: QueryHistoryInfo | undefined,
multiSelect: QueryHistoryInfo[] | undefined,
): {
finalSingleItem: QueryHistoryInfo;
finalSingleItem: QueryHistoryInfo | undefined;
finalMultiSelect: QueryHistoryInfo[];
} {
if (!singleItem && !multiSelect?.[0]) {
@@ -1325,7 +1322,7 @@ export class QueryHistoryManager extends DisposableObject {
}
return {
finalSingleItem: singleItem,
finalMultiSelect: multiSelect,
finalMultiSelect: multiSelect || [],
};
}

View File

@@ -1,105 +0,0 @@
import {
LocalQueryInfo,
CompletedQueryInfo,
InitialQueryInfo,
} from "../../query-results";
import { QueryEvaluationInfo } from "../../run-queries-shared";
import { QueryHistoryInfo } from "../query-history-info";
import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
import {
CompletedQueryInfoData,
QueryEvaluationInfoData,
InitialQueryInfoData,
LocalQueryDataItem,
} from "./local-query-data-item";
import { QueryHistoryDataItem } from "./query-history-data";
// Maps Query History Data Models to Domain Models
export function mapQueryHistoryToDomainModels(
queries: QueryHistoryDataItem[],
): QueryHistoryInfo[] {
return queries.map((d) => {
if (d.t === "variant-analysis") {
const query: VariantAnalysisHistoryItem = d;
return query;
} else if (d.t === "local") {
return mapLocalQueryDataItemToDomainModel(d);
}
throw Error(
`Unexpected or corrupted query history file. Unknown query history item: ${JSON.stringify(
d,
)}`,
);
});
}
function mapLocalQueryDataItemToDomainModel(
localQuery: LocalQueryDataItem,
): LocalQueryInfo {
return new LocalQueryInfo(
mapInitialQueryInfoDataToDomainModel(localQuery.initialInfo),
undefined,
localQuery.failureReason,
localQuery.completedQuery &&
mapCompletedQueryInfoDataToDomainModel(localQuery.completedQuery),
localQuery.evalLogLocation,
localQuery.evalLogSummaryLocation,
localQuery.jsonEvalLogSummaryLocation,
localQuery.evalLogSummarySymbolsLocation,
);
}
function mapCompletedQueryInfoDataToDomainModel(
completedQuery: CompletedQueryInfoData,
): CompletedQueryInfo {
return new CompletedQueryInfo(
mapQueryEvaluationInfoDataToDomainModel(completedQuery.query),
{
runId: completedQuery.result.runId,
queryId: completedQuery.result.queryId,
resultType: completedQuery.result.resultType,
evaluationTime: completedQuery.result.evaluationTime,
message: completedQuery.result.message,
logFileLocation: completedQuery.result.logFileLocation,
},
completedQuery.logFileLocation,
completedQuery.successful ?? completedQuery.sucessful,
completedQuery.message,
completedQuery.interpretedResultsSortState,
completedQuery.resultCount,
completedQuery.sortedResultsInfo,
);
}
function mapInitialQueryInfoDataToDomainModel(
initialInfo: InitialQueryInfoData,
): InitialQueryInfo {
return {
userSpecifiedLabel: initialInfo.userSpecifiedLabel,
queryText: initialInfo.queryText,
isQuickQuery: initialInfo.isQuickQuery,
isQuickEval: initialInfo.isQuickEval,
quickEvalPosition: initialInfo.quickEvalPosition,
queryPath: initialInfo.queryPath,
databaseInfo: {
databaseUri: initialInfo.databaseInfo.databaseUri,
name: initialInfo.databaseInfo.name,
},
start: new Date(initialInfo.start),
id: initialInfo.id,
};
}
function mapQueryEvaluationInfoDataToDomainModel(
evaluationInfo: QueryEvaluationInfoData,
): QueryEvaluationInfo {
return new QueryEvaluationInfo(
evaluationInfo.querySaveDir,
evaluationInfo.dbItemPath,
evaluationInfo.databaseHasMetadataFile,
evaluationInfo.quickEvalPosition,
evaluationInfo.metadata,
);
}

View File

@@ -1,90 +0,0 @@
import { assertNever } from "../../pure/helpers-pure";
import { LocalQueryInfo, InitialQueryInfo } from "../../query-results";
import { QueryEvaluationInfo } from "../../run-queries-shared";
import { QueryHistoryInfo } from "../query-history-info";
import {
LocalQueryDataItem,
InitialQueryInfoData,
QueryEvaluationInfoData,
} from "./local-query-data-item";
import { QueryHistoryDataItem } from "./query-history-data";
import { VariantAnalysisDataItem } from "./variant-analysis-data-item";
// Maps Query History Domain Models to Data Models
export function mapQueryHistoryToDataModels(
queries: QueryHistoryInfo[],
): QueryHistoryDataItem[] {
return queries.map((q) => {
if (q.t === "variant-analysis") {
const query: VariantAnalysisDataItem = q;
return query;
} else if (q.t === "local") {
return mapLocalQueryInfoToDataModel(q);
} else {
assertNever(q);
}
});
}
function mapLocalQueryInfoToDataModel(
query: LocalQueryInfo,
): LocalQueryDataItem {
return {
initialInfo: mapInitialQueryInfoToDataModel(query.initialInfo),
t: "local",
evalLogLocation: query.evalLogLocation,
evalLogSummaryLocation: query.evalLogSummaryLocation,
jsonEvalLogSummaryLocation: query.jsonEvalLogSummaryLocation,
evalLogSummarySymbolsLocation: query.evalLogSummarySymbolsLocation,
failureReason: query.failureReason,
completedQuery: query.completedQuery && {
query: mapQueryEvaluationInfoToDataModel(query.completedQuery.query),
result: {
runId: query.completedQuery.result.runId,
queryId: query.completedQuery.result.queryId,
resultType: query.completedQuery.result.resultType,
evaluationTime: query.completedQuery.result.evaluationTime,
message: query.completedQuery.result.message,
logFileLocation: query.completedQuery.result.logFileLocation,
},
logFileLocation: query.completedQuery.logFileLocation,
successful: query.completedQuery.successful,
message: query.completedQuery.message,
resultCount: query.completedQuery.resultCount,
sortedResultsInfo: query.completedQuery.sortedResultsInfo,
},
};
}
function mapInitialQueryInfoToDataModel(
localQueryInitialInfo: InitialQueryInfo,
): InitialQueryInfoData {
return {
userSpecifiedLabel: localQueryInitialInfo.userSpecifiedLabel,
queryText: localQueryInitialInfo.queryText,
isQuickQuery: localQueryInitialInfo.isQuickQuery,
isQuickEval: localQueryInitialInfo.isQuickEval,
quickEvalPosition: localQueryInitialInfo.quickEvalPosition,
queryPath: localQueryInitialInfo.queryPath,
databaseInfo: {
databaseUri: localQueryInitialInfo.databaseInfo.databaseUri,
name: localQueryInitialInfo.databaseInfo.name,
},
start: localQueryInitialInfo.start,
id: localQueryInitialInfo.id,
};
}
function mapQueryEvaluationInfoToDataModel(
queryEvaluationInfo: QueryEvaluationInfo,
): QueryEvaluationInfoData {
return {
querySaveDir: queryEvaluationInfo.querySaveDir,
dbItemPath: queryEvaluationInfo.dbItemPath,
databaseHasMetadataFile: queryEvaluationInfo.databaseHasMetadataFile,
quickEvalPosition: queryEvaluationInfo.quickEvalPosition,
metadata: queryEvaluationInfo.metadata,
resultsPaths: queryEvaluationInfo.resultsPaths,
};
}

View File

@@ -0,0 +1 @@
export * from "./query-history-store";

View File

@@ -1,100 +0,0 @@
export interface LocalQueryDataItem {
initialInfo: InitialQueryInfoData;
t: "local";
evalLogLocation?: string;
evalLogSummaryLocation?: string;
jsonEvalLogSummaryLocation?: string;
evalLogSummarySymbolsLocation?: string;
completedQuery?: CompletedQueryInfoData;
failureReason?: string;
}
export interface InitialQueryInfoData {
userSpecifiedLabel?: string;
queryText: string;
isQuickQuery: boolean;
isQuickEval: boolean;
quickEvalPosition?: PositionData;
queryPath: string;
databaseInfo: DatabaseInfoData;
start: Date;
id: string;
}
interface DatabaseInfoData {
name: string;
databaseUri: string;
}
interface PositionData {
line: number;
column: number;
endLine: number;
endColumn: number;
fileName: string;
}
export interface CompletedQueryInfoData {
query: QueryEvaluationInfoData;
message?: string;
successful?: boolean;
// There once was a typo in the data model, which is why we need to support both
sucessful?: boolean;
result: EvaluationResultData;
logFileLocation?: string;
resultCount: number;
sortedResultsInfo: Record<string, SortedResultSetInfo>;
interpretedResultsSortState?: InterpretedResultsSortState;
}
interface InterpretedResultsSortState {
sortBy: InterpretedResultsSortColumn;
sortDirection: SortDirection;
}
type InterpretedResultsSortColumn = "alert-message";
interface SortedResultSetInfo {
resultsPath: string;
sortState: RawResultsSortState;
}
interface RawResultsSortState {
columnIndex: number;
sortDirection: SortDirection;
}
enum SortDirection {
asc,
desc,
}
interface EvaluationResultData {
runId: number;
queryId: number;
resultType: number;
evaluationTime: number;
message?: string;
logFileLocation?: string;
}
export interface QueryEvaluationInfoData {
querySaveDir: string;
dbItemPath: string;
databaseHasMetadataFile: boolean;
quickEvalPosition?: PositionData;
metadata?: QueryMetadataData;
resultsPaths: {
resultsPath: string;
interpretedResultsPath: string;
};
}
interface QueryMetadataData {
name?: string;
description?: string;
id?: string;
kind?: string;
scored?: string;
}

View File

@@ -1,14 +0,0 @@
// Contains models and consts for the data we want to store in the query history store.
// Changes to these models should be done carefully and account for backwards compatibility of data.
import { LocalQueryDataItem } from "./local-query-data-item";
import { VariantAnalysisDataItem } from "./variant-analysis-data-item";
export const ALLOWED_QUERY_HISTORY_VERSIONS = [1, 2];
export interface QueryHistoryData {
version: number;
queries: QueryHistoryDataItem[];
}
export type QueryHistoryDataItem = LocalQueryDataItem | VariantAnalysisDataItem;

View File

@@ -0,0 +1,19 @@
import { assertNever } from "../../pure/helpers-pure";
import { QueryHistoryInfo } from "../query-history-info";
import { mapLocalQueryInfoToDto } from "./query-history-local-query-domain-mapper";
import { QueryHistoryItemDto } from "./query-history-dto";
import { mapQueryHistoryVariantAnalysisToDto } from "./query-history-variant-analysis-domain-mapper";
export function mapQueryHistoryToDto(
queries: QueryHistoryInfo[],
): QueryHistoryItemDto[] {
return queries.map((q) => {
if (q.t === "variant-analysis") {
return mapQueryHistoryVariantAnalysisToDto(q);
} else if (q.t === "local") {
return mapLocalQueryInfoToDto(q);
} else {
assertNever(q);
}
});
}

View File

@@ -0,0 +1,22 @@
import { QueryHistoryInfo } from "../query-history-info";
import { QueryHistoryItemDto } from "./query-history-dto";
import { mapQueryHistoryVariantAnalysisToDomainModel } from "./query-history-variant-analysis-dto-mapper";
import { mapLocalQueryItemToDomainModel } from "./query-history-local-query-dto-mapper";
export function mapQueryHistoryToDomainModel(
queries: QueryHistoryItemDto[],
): QueryHistoryInfo[] {
return queries.map((d) => {
if (d.t === "variant-analysis") {
return mapQueryHistoryVariantAnalysisToDomainModel(d);
} else if (d.t === "local") {
return mapLocalQueryItemToDomainModel(d);
}
throw Error(
`Unexpected or corrupted query history file. Unknown query history item: ${JSON.stringify(
d,
)}`,
);
});
}

View File

@@ -0,0 +1,14 @@
// Contains models and consts for the data we want to store in the query history store.
// Changes to these models should be done carefully and account for backwards compatibility of data.
import { QueryHistoryLocalQueryDto } from "./query-history-local-query-dto";
import { QueryHistoryVariantAnalysisDto } from "./query-history-variant-analysis-dto";
export interface QueryHistoryDto {
version: number;
queries: QueryHistoryItemDto[];
}
export type QueryHistoryItemDto =
| QueryHistoryLocalQueryDto
| QueryHistoryVariantAnalysisDto;

View File

@@ -0,0 +1,121 @@
import {
LocalQueryInfo,
InitialQueryInfo,
CompletedQueryInfo,
} from "../../query-results";
import { QueryEvaluationInfo } from "../../run-queries-shared";
import {
QueryHistoryLocalQueryDto,
InitialQueryInfoDto,
QueryEvaluationInfoDto,
CompletedQueryInfoDto,
SortedResultSetInfoDto,
SortDirectionDto,
} from "./query-history-local-query-dto";
import {
RawResultsSortState,
SortDirection,
SortedResultSetInfo,
} from "../../pure/interface-types";
export function mapLocalQueryInfoToDto(
query: LocalQueryInfo,
): QueryHistoryLocalQueryDto {
return {
initialInfo: mapInitialQueryInfoToDto(query.initialInfo),
t: "local",
evalLogLocation: query.evalLogLocation,
evalLogSummaryLocation: query.evalLogSummaryLocation,
jsonEvalLogSummaryLocation: query.jsonEvalLogSummaryLocation,
evalLogSummarySymbolsLocation: query.evalLogSummarySymbolsLocation,
failureReason: query.failureReason,
completedQuery:
query.completedQuery && mapCompletedQueryToDto(query.completedQuery),
};
}
function mapCompletedQueryToDto(
query: CompletedQueryInfo,
): CompletedQueryInfoDto {
const sortedResults = Object.fromEntries(
Object.entries(query.sortedResultsInfo).map(([key, value]) => {
return [key, mapSortedResultSetInfoToDto(value)];
}),
);
return {
query: mapQueryEvaluationInfoToDto(query.query),
result: {
runId: query.result.runId,
queryId: query.result.queryId,
resultType: query.result.resultType,
evaluationTime: query.result.evaluationTime,
message: query.result.message,
logFileLocation: query.result.logFileLocation,
},
logFileLocation: query.logFileLocation,
successful: query.successful,
message: query.message,
resultCount: query.resultCount,
sortedResultsInfo: sortedResults,
};
}
function mapSortDirectionToDto(sortDirection: SortDirection): SortDirectionDto {
switch (sortDirection) {
case SortDirection.asc:
return SortDirectionDto.asc;
case SortDirection.desc:
return SortDirectionDto.desc;
}
}
function mapRawResultsSortStateToDto(
sortState: RawResultsSortState,
): SortedResultSetInfoDto["sortState"] {
return {
columnIndex: sortState.columnIndex,
sortDirection: mapSortDirectionToDto(sortState.sortDirection),
};
}
function mapSortedResultSetInfoToDto(
resultSet: SortedResultSetInfo,
): SortedResultSetInfoDto {
return {
resultsPath: resultSet.resultsPath,
sortState: mapRawResultsSortStateToDto(resultSet.sortState),
};
}
function mapInitialQueryInfoToDto(
localQueryInitialInfo: InitialQueryInfo,
): InitialQueryInfoDto {
return {
userSpecifiedLabel: localQueryInitialInfo.userSpecifiedLabel,
queryText: localQueryInitialInfo.queryText,
isQuickQuery: localQueryInitialInfo.isQuickQuery,
isQuickEval: localQueryInitialInfo.isQuickEval,
quickEvalPosition: localQueryInitialInfo.quickEvalPosition,
queryPath: localQueryInitialInfo.queryPath,
databaseInfo: {
databaseUri: localQueryInitialInfo.databaseInfo.databaseUri,
name: localQueryInitialInfo.databaseInfo.name,
},
start: localQueryInitialInfo.start,
id: localQueryInitialInfo.id,
};
}
function mapQueryEvaluationInfoToDto(
queryEvaluationInfo: QueryEvaluationInfo,
): QueryEvaluationInfoDto {
return {
querySaveDir: queryEvaluationInfo.querySaveDir,
dbItemPath: queryEvaluationInfo.dbItemPath,
databaseHasMetadataFile: queryEvaluationInfo.databaseHasMetadataFile,
quickEvalPosition: queryEvaluationInfo.quickEvalPosition,
metadata: queryEvaluationInfo.metadata,
resultsPaths: queryEvaluationInfo.resultsPaths,
};
}

View File

@@ -0,0 +1,141 @@
import {
LocalQueryInfo,
CompletedQueryInfo,
InitialQueryInfo,
} from "../../query-results";
import { QueryEvaluationInfo } from "../../run-queries-shared";
import {
CompletedQueryInfoDto,
QueryEvaluationInfoDto,
InitialQueryInfoDto,
QueryHistoryLocalQueryDto,
SortDirectionDto,
InterpretedResultsSortStateDto,
SortedResultSetInfoDto,
RawResultsSortStateDto,
} from "./query-history-local-query-dto";
import {
InterpretedResultsSortState,
RawResultsSortState,
SortDirection,
SortedResultSetInfo,
} from "../../pure/interface-types";
export function mapLocalQueryItemToDomainModel(
localQuery: QueryHistoryLocalQueryDto,
): LocalQueryInfo {
return new LocalQueryInfo(
mapInitialQueryInfoToDomainModel(localQuery.initialInfo),
undefined,
localQuery.failureReason,
localQuery.completedQuery &&
mapCompletedQueryInfoToDomainModel(localQuery.completedQuery),
localQuery.evalLogLocation,
localQuery.evalLogSummaryLocation,
localQuery.jsonEvalLogSummaryLocation,
localQuery.evalLogSummarySymbolsLocation,
);
}
function mapCompletedQueryInfoToDomainModel(
completedQuery: CompletedQueryInfoDto,
): CompletedQueryInfo {
const sortState =
completedQuery.interpretedResultsSortState &&
mapSortStateToDomainModel(completedQuery.interpretedResultsSortState);
const sortedResults = Object.fromEntries(
Object.entries(completedQuery.sortedResultsInfo).map(([key, value]) => {
return [key, mapSortedResultSetInfoToDomainModel(value)];
}),
);
return new CompletedQueryInfo(
mapQueryEvaluationInfoToDomainModel(completedQuery.query),
{
runId: completedQuery.result.runId,
queryId: completedQuery.result.queryId,
resultType: completedQuery.result.resultType,
evaluationTime: completedQuery.result.evaluationTime,
message: completedQuery.result.message,
logFileLocation: completedQuery.result.logFileLocation,
},
completedQuery.logFileLocation,
completedQuery.successful ?? completedQuery.sucessful,
completedQuery.message,
sortState,
completedQuery.resultCount,
sortedResults,
);
}
function mapInitialQueryInfoToDomainModel(
initialInfo: InitialQueryInfoDto,
): InitialQueryInfo {
return {
userSpecifiedLabel: initialInfo.userSpecifiedLabel,
queryText: initialInfo.queryText,
isQuickQuery: initialInfo.isQuickQuery,
isQuickEval: initialInfo.isQuickEval,
quickEvalPosition: initialInfo.quickEvalPosition,
queryPath: initialInfo.queryPath,
databaseInfo: {
databaseUri: initialInfo.databaseInfo.databaseUri,
name: initialInfo.databaseInfo.name,
},
start: new Date(initialInfo.start),
id: initialInfo.id,
};
}
function mapQueryEvaluationInfoToDomainModel(
evaluationInfo: QueryEvaluationInfoDto,
): QueryEvaluationInfo {
return new QueryEvaluationInfo(
evaluationInfo.querySaveDir,
evaluationInfo.dbItemPath,
evaluationInfo.databaseHasMetadataFile,
evaluationInfo.quickEvalPosition,
evaluationInfo.metadata,
);
}
function mapSortDirectionToDomainModel(
sortDirection: SortDirectionDto,
): SortDirection {
switch (sortDirection) {
case SortDirectionDto.asc:
return SortDirection.asc;
case SortDirectionDto.desc:
return SortDirection.desc;
}
}
function mapSortStateToDomainModel(
sortState: InterpretedResultsSortStateDto,
): InterpretedResultsSortState {
return {
sortBy: sortState.sortBy,
sortDirection: mapSortDirectionToDomainModel(sortState.sortDirection),
};
}
function mapSortedResultSetInfoToDomainModel(
sortedResultSetInfo: SortedResultSetInfoDto,
): SortedResultSetInfo {
return {
resultsPath: sortedResultSetInfo.resultsPath,
sortState: mapRawResultsSortStateToDomainModel(
sortedResultSetInfo.sortState,
),
};
}
function mapRawResultsSortStateToDomainModel(
sortState: RawResultsSortStateDto,
): RawResultsSortState {
return {
columnIndex: sortState.columnIndex,
sortDirection: mapSortDirectionToDomainModel(sortState.sortDirection),
};
}

View File

@@ -0,0 +1,103 @@
// Contains models and consts for the data we want to store in the query history store.
// Changes to these models should be done carefully and account for backwards compatibility of data.
export interface QueryHistoryLocalQueryDto {
initialInfo: InitialQueryInfoDto;
t: "local";
evalLogLocation?: string;
evalLogSummaryLocation?: string;
jsonEvalLogSummaryLocation?: string;
evalLogSummarySymbolsLocation?: string;
completedQuery?: CompletedQueryInfoDto;
failureReason?: string;
}
export interface InitialQueryInfoDto {
userSpecifiedLabel?: string;
queryText: string;
isQuickQuery: boolean;
isQuickEval: boolean;
quickEvalPosition?: PositionDto;
queryPath: string;
databaseInfo: DatabaseInfoDto;
start: Date;
id: string;
}
interface DatabaseInfoDto {
name: string;
databaseUri: string;
}
interface PositionDto {
line: number;
column: number;
endLine: number;
endColumn: number;
fileName: string;
}
export interface CompletedQueryInfoDto {
query: QueryEvaluationInfoDto;
message?: string;
successful?: boolean;
// There once was a typo in the data model, which is why we need to support both
sucessful?: boolean;
result: EvaluationResultDto;
logFileLocation?: string;
resultCount: number;
sortedResultsInfo: Record<string, SortedResultSetInfoDto>;
interpretedResultsSortState?: InterpretedResultsSortStateDto;
}
export interface InterpretedResultsSortStateDto {
sortBy: InterpretedResultsSortColumnDto;
sortDirection: SortDirectionDto;
}
type InterpretedResultsSortColumnDto = "alert-message";
export interface SortedResultSetInfoDto {
resultsPath: string;
sortState: RawResultsSortStateDto;
}
export interface RawResultsSortStateDto {
columnIndex: number;
sortDirection: SortDirectionDto;
}
export enum SortDirectionDto {
asc,
desc,
}
interface EvaluationResultDto {
runId: number;
queryId: number;
resultType: number;
evaluationTime: number;
message?: string;
logFileLocation?: string;
}
export interface QueryEvaluationInfoDto {
querySaveDir: string;
dbItemPath: string;
databaseHasMetadataFile: boolean;
quickEvalPosition?: PositionDto;
metadata?: QueryMetadataDto;
resultsPaths: {
resultsPath: string;
interpretedResultsPath: string;
};
}
interface QueryMetadataDto {
name?: string;
description?: string;
id?: string;
kind?: string;
scored?: string;
}

View File

@@ -10,13 +10,11 @@ import {
} from "../../pure/helpers-pure";
import { QueryHistoryInfo } from "../query-history-info";
import { redactableError } from "../../pure/errors";
import {
ALLOWED_QUERY_HISTORY_VERSIONS,
QueryHistoryData,
QueryHistoryDataItem,
} from "./query-history-data";
import { mapQueryHistoryToDomainModels } from "./data-mapper";
import { mapQueryHistoryToDataModels } from "./domain-mapper";
import { QueryHistoryDto, QueryHistoryItemDto } from "./query-history-dto";
import { mapQueryHistoryToDomainModel } from "./query-history-dto-mapper";
import { mapQueryHistoryToDto } from "./query-history-domain-mapper";
const ALLOWED_QUERY_HISTORY_VERSIONS = [1, 2];
export async function readQueryHistoryFromFile(
fsPath: string,
@@ -26,7 +24,7 @@ export async function readQueryHistoryFromFile(
return [];
}
const obj: QueryHistoryData = await readJson(fsPath, {
const obj: QueryHistoryDto = await readJson(fsPath, {
encoding: "utf8",
});
@@ -40,21 +38,21 @@ export async function readQueryHistoryFromFile(
const queries = obj.queries;
// Remove remote queries, which are not supported anymore.
const parsedQueries = queries.filter(
(q: QueryHistoryDataItem | { t: "remote" }) => q.t !== "remote",
(q: QueryHistoryItemDto | { t: "remote" }) => q.t !== "remote",
);
// Map the data models to the domain models.
const domainModels: QueryHistoryInfo[] =
mapQueryHistoryToDomainModels(parsedQueries);
mapQueryHistoryToDomainModel(parsedQueries);
// filter out queries that have been deleted on disk
// Filter out queries that have been deleted on disk
// most likely another workspace has deleted them because the
// queries aged out.
const filteredDomainModels: Promise<QueryHistoryInfo[]> = asyncFilter(
domainModels,
async (q) => {
if (q.t === "variant-analysis") {
// the query history store doesn't know where variant analysises are
// The query history store doesn't know where variant analysises are
// stored so we need to assume here that they exist. We check later
// to see if they exist on disk.
return true;
@@ -72,7 +70,7 @@ export async function readQueryHistoryFromFile(
fullMessage: `Error loading query history.\n${getErrorStack(e)}`,
},
);
// since the query history is invalid, it should be deleted so this error does not happen on next startup.
// Since the query history is invalid, it should be deleted so this error does not happen on next startup.
await remove(fsPath);
return [];
}
@@ -95,13 +93,13 @@ export async function writeQueryHistoryToFile(
if (!(await pathExists(fsPath))) {
await mkdir(dirname(fsPath), { recursive: true });
}
// remove incomplete local queries since they cannot be recreated on restart
// Remove incomplete local queries since they cannot be recreated on restart
const filteredQueries = queries.filter((q) =>
q.t === "local" ? q.completedQuery !== undefined : true,
);
// map domain model queries to data model
const queryHistoryData = mapQueryHistoryToDataModels(filteredQueries);
// Map domain model queries to data model
const queryHistoryData = mapQueryHistoryToDto(filteredQueries);
const data = JSON.stringify(
{

View File

@@ -0,0 +1,235 @@
import {
QueryHistoryVariantAnalysisDto,
QueryLanguageDto,
QueryStatusDto,
VariantAnalysisDto,
VariantAnalysisFailureReasonDto,
VariantAnalysisRepoStatusDto,
VariantAnalysisScannedRepositoryDto,
VariantAnalysisSkippedRepositoriesDto,
VariantAnalysisSkippedRepositoryDto,
VariantAnalysisSkippedRepositoryGroupDto,
VariantAnalysisStatusDto,
} from "./query-history-variant-analysis-dto";
import {
VariantAnalysis,
VariantAnalysisFailureReason,
VariantAnalysisRepoStatus,
VariantAnalysisScannedRepository,
VariantAnalysisSkippedRepositories,
VariantAnalysisSkippedRepository,
VariantAnalysisSkippedRepositoryGroup,
VariantAnalysisStatus,
} from "../../variant-analysis/shared/variant-analysis";
import { assertNever } from "../../pure/helpers-pure";
import { QueryLanguage } from "../../common/query-language";
import { QueryStatus } from "../../query-status";
import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
export function mapQueryHistoryVariantAnalysisToDto(
item: VariantAnalysisHistoryItem,
): QueryHistoryVariantAnalysisDto {
return {
t: "variant-analysis",
failureReason: item.failureReason,
resultCount: item.resultCount,
status: mapQueryStatusToDto(item.status),
completed: item.completed,
variantAnalysis: mapVariantAnalysisDtoToDto(item.variantAnalysis),
userSpecifiedLabel: item.userSpecifiedLabel,
};
}
function mapVariantAnalysisDtoToDto(
variantAnalysis: VariantAnalysis,
): VariantAnalysisDto {
return {
id: variantAnalysis.id,
controllerRepo: {
id: variantAnalysis.controllerRepo.id,
fullName: variantAnalysis.controllerRepo.fullName,
private: variantAnalysis.controllerRepo.private,
},
query: {
name: variantAnalysis.query.name,
filePath: variantAnalysis.query.filePath,
language: mapQueryLanguageToDto(variantAnalysis.query.language),
text: variantAnalysis.query.text,
},
databases: {
repositories: variantAnalysis.databases.repositories,
repositoryLists: variantAnalysis.databases.repositoryLists,
repositoryOwners: variantAnalysis.databases.repositoryOwners,
},
createdAt: variantAnalysis.createdAt,
updatedAt: variantAnalysis.updatedAt,
executionStartTime: variantAnalysis.executionStartTime,
status: mapVariantAnalysisStatusToDto(variantAnalysis.status),
completedAt: variantAnalysis.completedAt,
actionsWorkflowRunId: variantAnalysis.actionsWorkflowRunId,
failureReason:
variantAnalysis.failureReason &&
mapVariantAnalysisFailureReasonToDto(variantAnalysis.failureReason),
scannedRepos:
variantAnalysis.scannedRepos &&
mapVariantAnalysisScannedRepositoriesToDto(variantAnalysis.scannedRepos),
skippedRepos:
variantAnalysis.skippedRepos &&
mapVariantAnalysisSkippedRepositoriesToDto(variantAnalysis.skippedRepos),
};
}
function mapVariantAnalysisScannedRepositoriesToDto(
repos: VariantAnalysisScannedRepository[],
): VariantAnalysisScannedRepositoryDto[] {
return repos.map(mapVariantAnalysisScannedRepositoryToDto);
}
function mapVariantAnalysisScannedRepositoryToDto(
repo: VariantAnalysisScannedRepository,
): VariantAnalysisScannedRepositoryDto {
return {
repository: {
id: repo.repository.id,
fullName: repo.repository.fullName,
private: repo.repository.private,
stargazersCount: repo.repository.stargazersCount,
updatedAt: repo.repository.updatedAt,
},
analysisStatus: mapVariantAnalysisRepoStatusToDto(repo.analysisStatus),
resultCount: repo.resultCount,
artifactSizeInBytes: repo.artifactSizeInBytes,
failureMessage: repo.failureMessage,
};
}
function mapVariantAnalysisSkippedRepositoriesToDto(
repos: VariantAnalysisSkippedRepositories,
): VariantAnalysisSkippedRepositoriesDto {
return {
accessMismatchRepos:
repos.accessMismatchRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDto(repos.accessMismatchRepos),
notFoundRepos:
repos.notFoundRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDto(repos.notFoundRepos),
noCodeqlDbRepos:
repos.noCodeqlDbRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDto(repos.noCodeqlDbRepos),
overLimitRepos:
repos.overLimitRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDto(repos.overLimitRepos),
};
}
function mapVariantAnalysisSkippedRepositoryGroupToDto(
repoGroup: VariantAnalysisSkippedRepositoryGroup,
): VariantAnalysisSkippedRepositoryGroupDto {
return {
repositoryCount: repoGroup.repositoryCount,
repositories: repoGroup.repositories.map(
mapVariantAnalysisSkippedRepositoryToDto,
),
};
}
function mapVariantAnalysisSkippedRepositoryToDto(
repo: VariantAnalysisSkippedRepository,
): VariantAnalysisSkippedRepositoryDto {
return {
id: repo.id,
fullName: repo.fullName,
private: repo.private,
stargazersCount: repo.stargazersCount,
updatedAt: repo.updatedAt,
};
}
function mapVariantAnalysisFailureReasonToDto(
failureReason: VariantAnalysisFailureReason,
): VariantAnalysisFailureReasonDto {
switch (failureReason) {
case VariantAnalysisFailureReason.NoReposQueried:
return VariantAnalysisFailureReasonDto.NoReposQueried;
case VariantAnalysisFailureReason.ActionsWorkflowRunFailed:
return VariantAnalysisFailureReasonDto.ActionsWorkflowRunFailed;
case VariantAnalysisFailureReason.InternalError:
return VariantAnalysisFailureReasonDto.InternalError;
default:
assertNever(failureReason);
}
}
function mapVariantAnalysisRepoStatusToDto(
status: VariantAnalysisRepoStatus,
): VariantAnalysisRepoStatusDto {
switch (status) {
case VariantAnalysisRepoStatus.Pending:
return VariantAnalysisRepoStatusDto.Pending;
case VariantAnalysisRepoStatus.InProgress:
return VariantAnalysisRepoStatusDto.InProgress;
case VariantAnalysisRepoStatus.Succeeded:
return VariantAnalysisRepoStatusDto.Succeeded;
case VariantAnalysisRepoStatus.Failed:
return VariantAnalysisRepoStatusDto.Failed;
case VariantAnalysisRepoStatus.Canceled:
return VariantAnalysisRepoStatusDto.Canceled;
case VariantAnalysisRepoStatus.TimedOut:
return VariantAnalysisRepoStatusDto.TimedOut;
default:
assertNever(status);
}
}
function mapVariantAnalysisStatusToDto(
status: VariantAnalysisStatus,
): VariantAnalysisStatusDto {
switch (status) {
case VariantAnalysisStatus.InProgress:
return VariantAnalysisStatusDto.InProgress;
case VariantAnalysisStatus.Succeeded:
return VariantAnalysisStatusDto.Succeeded;
case VariantAnalysisStatus.Failed:
return VariantAnalysisStatusDto.Failed;
case VariantAnalysisStatus.Canceled:
return VariantAnalysisStatusDto.Canceled;
default:
assertNever(status);
}
}
function mapQueryLanguageToDto(language: QueryLanguage): QueryLanguageDto {
switch (language) {
case QueryLanguage.CSharp:
return QueryLanguageDto.CSharp;
case QueryLanguage.Cpp:
return QueryLanguageDto.Cpp;
case QueryLanguage.Go:
return QueryLanguageDto.Go;
case QueryLanguage.Java:
return QueryLanguageDto.Java;
case QueryLanguage.Javascript:
return QueryLanguageDto.Javascript;
case QueryLanguage.Python:
return QueryLanguageDto.Python;
case QueryLanguage.Ruby:
return QueryLanguageDto.Ruby;
case QueryLanguage.Swift:
return QueryLanguageDto.Swift;
default:
assertNever(language);
}
}
function mapQueryStatusToDto(status: QueryStatus): QueryStatusDto {
switch (status) {
case QueryStatus.InProgress:
return QueryStatusDto.InProgress;
case QueryStatus.Completed:
return QueryStatusDto.Completed;
case QueryStatus.Failed:
return QueryStatusDto.Failed;
default:
assertNever(status);
}
}

View File

@@ -0,0 +1,253 @@
import {
QueryHistoryVariantAnalysisDto,
QueryLanguageDto,
QueryStatusDto,
VariantAnalysisDto,
VariantAnalysisFailureReasonDto,
VariantAnalysisRepoStatusDto,
VariantAnalysisScannedRepositoryDto,
VariantAnalysisSkippedRepositoriesDto,
VariantAnalysisSkippedRepositoryDto,
VariantAnalysisSkippedRepositoryGroupDto,
VariantAnalysisStatusDto,
} from "./query-history-variant-analysis-dto";
import {
VariantAnalysis,
VariantAnalysisFailureReason,
VariantAnalysisRepoStatus,
VariantAnalysisScannedRepository,
VariantAnalysisSkippedRepositories,
VariantAnalysisSkippedRepository,
VariantAnalysisSkippedRepositoryGroup,
VariantAnalysisStatus,
} from "../../variant-analysis/shared/variant-analysis";
import { assertNever } from "../../pure/helpers-pure";
import { QueryLanguage } from "../../common/query-language";
import { QueryStatus } from "../../query-status";
import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
export function mapQueryHistoryVariantAnalysisToDomainModel(
item: QueryHistoryVariantAnalysisDto,
): VariantAnalysisHistoryItem {
return {
t: "variant-analysis",
failureReason: item.failureReason,
resultCount: item.resultCount,
status: mapQueryStatusToDomainModel(item.status),
completed: item.completed,
variantAnalysis: mapVariantAnalysisToDomainModel(item.variantAnalysis),
userSpecifiedLabel: item.userSpecifiedLabel,
};
}
function mapVariantAnalysisToDomainModel(
variantAnalysis: VariantAnalysisDto,
): VariantAnalysis {
return {
id: variantAnalysis.id,
controllerRepo: {
id: variantAnalysis.controllerRepo.id,
fullName: variantAnalysis.controllerRepo.fullName,
private: variantAnalysis.controllerRepo.private,
},
query: {
name: variantAnalysis.query.name,
filePath: variantAnalysis.query.filePath,
language: mapQueryLanguageToDomainModel(variantAnalysis.query.language),
text: variantAnalysis.query.text,
},
databases: {
repositories: variantAnalysis.databases.repositories,
repositoryLists: variantAnalysis.databases.repositoryLists,
repositoryOwners: variantAnalysis.databases.repositoryOwners,
},
createdAt: variantAnalysis.createdAt,
updatedAt: variantAnalysis.updatedAt,
executionStartTime: variantAnalysis.executionStartTime,
status: mapVariantAnalysisStatusToDomainModel(variantAnalysis.status),
completedAt: variantAnalysis.completedAt,
actionsWorkflowRunId: variantAnalysis.actionsWorkflowRunId,
failureReason:
variantAnalysis.failureReason &&
mapVariantAnalysisFailureReasonToDomainModel(
variantAnalysis.failureReason,
),
scannedRepos:
variantAnalysis.scannedRepos &&
mapVariantAnalysisScannedRepositoriesToDomainModel(
variantAnalysis.scannedRepos,
),
skippedRepos:
variantAnalysis.skippedRepos &&
mapVariantAnalysisSkippedRepositoriesToDomainModel(
variantAnalysis.skippedRepos,
),
};
}
function mapVariantAnalysisScannedRepositoriesToDomainModel(
repos: VariantAnalysisScannedRepositoryDto[],
): VariantAnalysisScannedRepository[] {
return repos.map(mapVariantAnalysisScannedRepositoryToDomainModel);
}
function mapVariantAnalysisScannedRepositoryToDomainModel(
repo: VariantAnalysisScannedRepositoryDto,
): VariantAnalysisScannedRepository {
return {
repository: {
id: repo.repository.id,
fullName: repo.repository.fullName,
private: repo.repository.private,
stargazersCount: repo.repository.stargazersCount,
updatedAt: repo.repository.updatedAt,
},
analysisStatus: mapVariantAnalysisRepoStatusToDomainModel(
repo.analysisStatus,
),
resultCount: repo.resultCount,
artifactSizeInBytes: repo.artifactSizeInBytes,
failureMessage: repo.failureMessage,
};
}
function mapVariantAnalysisSkippedRepositoriesToDomainModel(
repos: VariantAnalysisSkippedRepositoriesDto,
): VariantAnalysisSkippedRepositories {
return {
accessMismatchRepos:
repos.accessMismatchRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
repos.accessMismatchRepos,
),
notFoundRepos:
repos.notFoundRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
repos.notFoundRepos,
),
noCodeqlDbRepos:
repos.noCodeqlDbRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
repos.noCodeqlDbRepos,
),
overLimitRepos:
repos.overLimitRepos &&
mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
repos.overLimitRepos,
),
};
}
function mapVariantAnalysisSkippedRepositoryGroupToDomainModel(
repoGroup: VariantAnalysisSkippedRepositoryGroupDto,
): VariantAnalysisSkippedRepositoryGroup {
return {
repositoryCount: repoGroup.repositoryCount,
repositories: repoGroup.repositories.map(
mapVariantAnalysisSkippedRepositoryToDomainModel,
),
};
}
function mapVariantAnalysisSkippedRepositoryToDomainModel(
repo: VariantAnalysisSkippedRepositoryDto,
): VariantAnalysisSkippedRepository {
return {
id: repo.id,
fullName: repo.fullName,
private: repo.private,
stargazersCount: repo.stargazersCount,
updatedAt: repo.updatedAt,
};
}
function mapVariantAnalysisFailureReasonToDomainModel(
failureReason: VariantAnalysisFailureReasonDto,
): VariantAnalysisFailureReason {
switch (failureReason) {
case VariantAnalysisFailureReasonDto.NoReposQueried:
return VariantAnalysisFailureReason.NoReposQueried;
case VariantAnalysisFailureReasonDto.ActionsWorkflowRunFailed:
return VariantAnalysisFailureReason.ActionsWorkflowRunFailed;
case VariantAnalysisFailureReasonDto.InternalError:
return VariantAnalysisFailureReason.InternalError;
default:
assertNever(failureReason);
}
}
function mapVariantAnalysisRepoStatusToDomainModel(
status: VariantAnalysisRepoStatusDto,
): VariantAnalysisRepoStatus {
switch (status) {
case VariantAnalysisRepoStatusDto.Pending:
return VariantAnalysisRepoStatus.Pending;
case VariantAnalysisRepoStatusDto.InProgress:
return VariantAnalysisRepoStatus.InProgress;
case VariantAnalysisRepoStatusDto.Succeeded:
return VariantAnalysisRepoStatus.Succeeded;
case VariantAnalysisRepoStatusDto.Failed:
return VariantAnalysisRepoStatus.Failed;
case VariantAnalysisRepoStatusDto.Canceled:
return VariantAnalysisRepoStatus.Canceled;
case VariantAnalysisRepoStatusDto.TimedOut:
return VariantAnalysisRepoStatus.TimedOut;
default:
assertNever(status);
}
}
function mapVariantAnalysisStatusToDomainModel(
status: VariantAnalysisStatusDto,
): VariantAnalysisStatus {
switch (status) {
case VariantAnalysisStatusDto.InProgress:
return VariantAnalysisStatus.InProgress;
case VariantAnalysisStatusDto.Succeeded:
return VariantAnalysisStatus.Succeeded;
case VariantAnalysisStatusDto.Failed:
return VariantAnalysisStatus.Failed;
case VariantAnalysisStatusDto.Canceled:
return VariantAnalysisStatus.Canceled;
default:
assertNever(status);
}
}
function mapQueryLanguageToDomainModel(
language: QueryLanguageDto,
): QueryLanguage {
switch (language) {
case QueryLanguageDto.CSharp:
return QueryLanguage.CSharp;
case QueryLanguageDto.Cpp:
return QueryLanguage.Cpp;
case QueryLanguageDto.Go:
return QueryLanguage.Go;
case QueryLanguageDto.Java:
return QueryLanguage.Java;
case QueryLanguageDto.Javascript:
return QueryLanguage.Javascript;
case QueryLanguageDto.Python:
return QueryLanguage.Python;
case QueryLanguageDto.Ruby:
return QueryLanguage.Ruby;
case QueryLanguageDto.Swift:
return QueryLanguage.Swift;
default:
assertNever(language);
}
}
function mapQueryStatusToDomainModel(status: QueryStatusDto): QueryStatus {
switch (status) {
case QueryStatusDto.InProgress:
return QueryStatus.InProgress;
case QueryStatusDto.Completed:
return QueryStatus.Completed;
case QueryStatusDto.Failed:
return QueryStatus.Failed;
default:
assertNever(status);
}
}

View File

@@ -0,0 +1,114 @@
// Contains models and consts for the data we want to store in the query history store.
// Changes to these models should be done carefully and account for backwards compatibility of data.
export interface QueryHistoryVariantAnalysisDto {
readonly t: "variant-analysis";
failureReason?: string;
resultCount?: number;
status: QueryStatusDto;
completed: boolean;
variantAnalysis: VariantAnalysisDto;
userSpecifiedLabel?: string;
}
export interface VariantAnalysisDto {
id: number;
controllerRepo: {
id: number;
fullName: string;
private: boolean;
};
query: {
name: string;
filePath: string;
language: QueryLanguageDto;
text: string;
};
databases: {
repositories?: string[];
repositoryLists?: string[];
repositoryOwners?: string[];
};
createdAt: string;
updatedAt: string;
executionStartTime: number;
status: VariantAnalysisStatusDto;
completedAt?: string;
actionsWorkflowRunId?: number;
failureReason?: VariantAnalysisFailureReasonDto;
scannedRepos?: VariantAnalysisScannedRepositoryDto[];
skippedRepos?: VariantAnalysisSkippedRepositoriesDto;
}
export interface VariantAnalysisScannedRepositoryDto {
repository: {
id: number;
fullName: string;
private: boolean;
stargazersCount: number;
updatedAt: string | null;
};
analysisStatus: VariantAnalysisRepoStatusDto;
resultCount?: number;
artifactSizeInBytes?: number;
failureMessage?: string;
}
export interface VariantAnalysisSkippedRepositoriesDto {
accessMismatchRepos?: VariantAnalysisSkippedRepositoryGroupDto;
notFoundRepos?: VariantAnalysisSkippedRepositoryGroupDto;
noCodeqlDbRepos?: VariantAnalysisSkippedRepositoryGroupDto;
overLimitRepos?: VariantAnalysisSkippedRepositoryGroupDto;
}
export interface VariantAnalysisSkippedRepositoryGroupDto {
repositoryCount: number;
repositories: VariantAnalysisSkippedRepositoryDto[];
}
export interface VariantAnalysisSkippedRepositoryDto {
id?: number;
fullName: string;
private?: boolean;
stargazersCount?: number;
updatedAt?: string | null;
}
export enum VariantAnalysisFailureReasonDto {
NoReposQueried = "noReposQueried",
ActionsWorkflowRunFailed = "actionsWorkflowRunFailed",
InternalError = "internalError",
}
export enum VariantAnalysisRepoStatusDto {
Pending = "pending",
InProgress = "inProgress",
Succeeded = "succeeded",
Failed = "failed",
Canceled = "canceled",
TimedOut = "timedOut",
}
export enum VariantAnalysisStatusDto {
InProgress = "inProgress",
Succeeded = "succeeded",
Failed = "failed",
Canceled = "canceled",
}
export enum QueryLanguageDto {
CSharp = "csharp",
Cpp = "cpp",
Go = "go",
Java = "java",
Javascript = "javascript",
Python = "python",
Ruby = "ruby",
Swift = "swift",
}
export enum QueryStatusDto {
InProgress = "InProgress",
Completed = "Completed",
Failed = "Failed",
}

View File

@@ -1,83 +0,0 @@
import { QueryLanguage } from "../../common/query-language";
import { QueryStatus } from "../../query-status";
import {
VariantAnalysisFailureReason,
VariantAnalysisRepoStatus,
VariantAnalysisStatus,
} from "../../variant-analysis/shared/variant-analysis";
// Data Model for Variant Analysis Query History Items
// All data points are modelled, except enums.
export interface VariantAnalysisDataItem {
readonly t: "variant-analysis";
failureReason?: string;
resultCount?: number;
status: QueryStatus;
completed: boolean;
variantAnalysis: VariantAnalysisQueryHistoryData;
userSpecifiedLabel?: string;
}
export interface VariantAnalysisQueryHistoryData {
id: number;
controllerRepo: {
id: number;
fullName: string;
private: boolean;
};
query: {
name: string;
filePath: string;
language: QueryLanguage;
text: string;
};
databases: {
repositories?: string[];
repositoryLists?: string[];
repositoryOwners?: string[];
};
createdAt: string;
updatedAt: string;
executionStartTime: number;
status: VariantAnalysisStatus;
completedAt?: string;
actionsWorkflowRunId?: number;
failureReason?: VariantAnalysisFailureReason;
scannedRepos?: VariantAnalysisScannedRepositoryData[];
skippedRepos?: VariantAnalysisSkippedRepositoriesData;
}
export interface VariantAnalysisScannedRepositoryData {
repository: {
id: number;
fullName: string;
private: boolean;
stargazersCount: number;
updatedAt: string | null;
};
analysisStatus: VariantAnalysisRepoStatus;
resultCount?: number;
artifactSizeInBytes?: number;
failureMessage?: string;
}
export interface VariantAnalysisSkippedRepositoriesData {
accessMismatchRepos?: VariantAnalysisSkippedRepositoryGroupData;
notFoundRepos?: VariantAnalysisSkippedRepositoryGroupData;
noCodeqlDbRepos?: VariantAnalysisSkippedRepositoryGroupData;
overLimitRepos?: VariantAnalysisSkippedRepositoryGroupData;
}
export interface VariantAnalysisSkippedRepositoryGroupData {
repositoryCount: number;
repositories: VariantAnalysisSkippedRepositoryData[];
}
export interface VariantAnalysisSkippedRepositoryData {
id?: number;
fullName: string;
private?: boolean;
stargazersCount?: number;
updatedAt?: string | null;
}

View File

@@ -261,8 +261,12 @@ export class SkeletonQueryWizard {
): Promise<DatabaseItem | undefined> {
const dbItems = databaseItems || [];
const dbs = dbItems.filter(
(db) => db.language === language && db.name === databaseNwo,
(db) =>
db.language === language &&
db.name === databaseNwo &&
db.error === undefined,
);
if (dbs.length === 0) {
return undefined;
}
@@ -274,7 +278,9 @@ export class SkeletonQueryWizard {
databaseItems: readonly DatabaseItem[],
): Promise<DatabaseItem | undefined> {
const dbItems = databaseItems || [];
const dbs = dbItems.filter((db) => db.language === language);
const dbs = dbItems.filter(
(db) => db.language === language && db.error === undefined,
);
if (dbs.length === 0) {
return undefined;
}

View File

@@ -0,0 +1 @@
export * from "./repo-states-store";

View File

@@ -0,0 +1,46 @@
import { assertNever } from "../../pure/helpers-pure";
import {
VariantAnalysisScannedRepositoryState,
VariantAnalysisScannedRepositoryDownloadStatus,
} from "../shared/variant-analysis";
import {
VariantAnalysisScannedRepositoryStateDto,
VariantAnalysisScannedRepositoryDownloadDto,
} from "./repo-states-dto";
export function mapRepoStatesToDomainModel(
repoStates: Record<number, VariantAnalysisScannedRepositoryStateDto>,
): Record<number, VariantAnalysisScannedRepositoryState> {
return Object.fromEntries(
Object.entries(repoStates).map(([key, value]) => {
return [key, mapRepoStateToDomainModel(value)];
}),
);
}
function mapRepoStateToDomainModel(
repoState: VariantAnalysisScannedRepositoryStateDto,
): VariantAnalysisScannedRepositoryState {
return {
repositoryId: repoState.repositoryId,
downloadStatus: mapDownloadStatusToDomainModel(repoState.downloadStatus),
downloadPercentage: repoState.downloadPercentage,
};
}
function mapDownloadStatusToDomainModel(
downloadedStatus: VariantAnalysisScannedRepositoryDownloadDto,
) {
switch (downloadedStatus) {
case VariantAnalysisScannedRepositoryDownloadDto.Pending:
return VariantAnalysisScannedRepositoryDownloadStatus.Pending;
case VariantAnalysisScannedRepositoryDownloadDto.InProgress:
return VariantAnalysisScannedRepositoryDownloadStatus.InProgress;
case VariantAnalysisScannedRepositoryDownloadDto.Succeeded:
return VariantAnalysisScannedRepositoryDownloadStatus.Succeeded;
case VariantAnalysisScannedRepositoryDownloadDto.Failed:
return VariantAnalysisScannedRepositoryDownloadStatus.Failed;
default:
assertNever(downloadedStatus);
}
}

View File

@@ -0,0 +1,46 @@
import { assertNever } from "../../pure/helpers-pure";
import {
VariantAnalysisScannedRepositoryDownloadStatus,
VariantAnalysisScannedRepositoryState,
} from "../shared/variant-analysis";
import {
VariantAnalysisScannedRepositoryDownloadDto,
VariantAnalysisScannedRepositoryStateDto,
} from "./repo-states-dto";
export function mapRepoStatesToDto(
repoStates: Record<number, VariantAnalysisScannedRepositoryState>,
): Record<number, VariantAnalysisScannedRepositoryStateDto> {
return Object.fromEntries(
Object.entries(repoStates).map(([key, value]) => {
return [key, mapRepoStateToDto(value)];
}),
);
}
function mapRepoStateToDto(
repoState: VariantAnalysisScannedRepositoryState,
): VariantAnalysisScannedRepositoryStateDto {
return {
repositoryId: repoState.repositoryId,
downloadStatus: mapDownloadStatusToDto(repoState.downloadStatus),
downloadPercentage: repoState.downloadPercentage,
};
}
function mapDownloadStatusToDto(
downloadedStatus: VariantAnalysisScannedRepositoryDownloadStatus,
) {
switch (downloadedStatus) {
case VariantAnalysisScannedRepositoryDownloadStatus.Pending:
return VariantAnalysisScannedRepositoryDownloadDto.Pending;
case VariantAnalysisScannedRepositoryDownloadStatus.InProgress:
return VariantAnalysisScannedRepositoryDownloadDto.InProgress;
case VariantAnalysisScannedRepositoryDownloadStatus.Succeeded:
return VariantAnalysisScannedRepositoryDownloadDto.Succeeded;
case VariantAnalysisScannedRepositoryDownloadStatus.Failed:
return VariantAnalysisScannedRepositoryDownloadDto.Failed;
default:
assertNever(downloadedStatus);
}
}

View File

@@ -0,0 +1,12 @@
export interface VariantAnalysisScannedRepositoryStateDto {
repositoryId: number;
downloadStatus: VariantAnalysisScannedRepositoryDownloadDto;
downloadPercentage?: number;
}
export enum VariantAnalysisScannedRepositoryDownloadDto {
Pending = "pending",
InProgress = "inProgress",
Succeeded = "succeeded",
Failed = "failed",
}

View File

@@ -1,8 +1,8 @@
import { outputJson, readJson } from "fs-extra";
import { VariantAnalysisScannedRepositoryState } from "../shared/variant-analysis";
import { VariantAnalysisScannedRepositoryStateData } from "./repo-states-data-types";
import { mapRepoStateToData } from "./repo-states-to-data-mapper";
import { mapRepoStateToDomain } from "./repo-states-to-domain-mapper";
import { VariantAnalysisScannedRepositoryStateDto } from "./repo-states-dto";
import { mapRepoStatesToDto } from "./repo-states-dto-mapper";
import { mapRepoStatesToDomainModel } from "./repo-states-domain-mapper";
export const REPO_STATES_FILENAME = "repo_states.json";
@@ -10,13 +10,7 @@ export async function writeRepoStates(
storagePath: string,
repoStates: Record<number, VariantAnalysisScannedRepositoryState>,
): Promise<void> {
// Map from repoStates Domain type to the repoStates Data type
const repoStatesData = Object.fromEntries(
Object.entries(repoStates).map(([key, value]) => {
return [key, mapRepoStateToData(value)];
}),
);
const repoStatesData = mapRepoStatesToDto(repoStates);
await outputJson(storagePath, repoStatesData);
}
@@ -26,15 +20,10 @@ export async function readRepoStates(
try {
const repoStatesData: Record<
number,
VariantAnalysisScannedRepositoryStateData
VariantAnalysisScannedRepositoryStateDto
> = await readJson(storagePath);
// Map from repoStates Data type to the repoStates Domain type
const repoStates = Object.fromEntries(
Object.entries(repoStatesData).map(([key, value]) => {
return [key, mapRepoStateToDomain(value)];
}),
);
const repoStates = mapRepoStatesToDomainModel(repoStatesData);
return repoStates;
} catch (e) {

View File

@@ -0,0 +1 @@
export * from "./repo-tasks-store";

View File

@@ -4,12 +4,12 @@ import {
VariantAnalysisRepoStatus,
} from "../shared/variant-analysis";
import {
VariantAnalysisRepositoryTaskData,
VariantAnalysisRepoStatusData,
} from "./repo-task-data-types";
VariantAnalysisRepositoryTaskDto,
VariantAnalysisRepoStatusDto,
} from "./repo-tasks-dto";
export function mapRepoTaskToDomain(
repoTask: VariantAnalysisRepositoryTaskData,
export function mapRepoTaskToDomainModel(
repoTask: VariantAnalysisRepositoryTaskDto,
): VariantAnalysisRepositoryTask {
return {
repository: {
@@ -17,7 +17,9 @@ export function mapRepoTaskToDomain(
fullName: repoTask.repository.fullName,
private: repoTask.repository.private,
},
analysisStatus: mapRepoTaskAnalysisStatusToDomain(repoTask.analysisStatus),
analysisStatus: mapRepoTaskAnalysisStatusToDomainModel(
repoTask.analysisStatus,
),
resultCount: repoTask.resultCount,
artifactSizeInBytes: repoTask.artifactSizeInBytes,
failureMessage: repoTask.failureMessage,
@@ -27,21 +29,21 @@ export function mapRepoTaskToDomain(
};
}
function mapRepoTaskAnalysisStatusToDomain(
analysisStatus: VariantAnalysisRepoStatusData,
function mapRepoTaskAnalysisStatusToDomainModel(
analysisStatus: VariantAnalysisRepoStatusDto,
): VariantAnalysisRepoStatus {
switch (analysisStatus) {
case VariantAnalysisRepoStatusData.Pending:
case VariantAnalysisRepoStatusDto.Pending:
return VariantAnalysisRepoStatus.Pending;
case VariantAnalysisRepoStatusData.InProgress:
case VariantAnalysisRepoStatusDto.InProgress:
return VariantAnalysisRepoStatus.InProgress;
case VariantAnalysisRepoStatusData.Succeeded:
case VariantAnalysisRepoStatusDto.Succeeded:
return VariantAnalysisRepoStatus.Succeeded;
case VariantAnalysisRepoStatusData.Failed:
case VariantAnalysisRepoStatusDto.Failed:
return VariantAnalysisRepoStatus.Failed;
case VariantAnalysisRepoStatusData.Canceled:
case VariantAnalysisRepoStatusDto.Canceled:
return VariantAnalysisRepoStatus.Canceled;
case VariantAnalysisRepoStatusData.TimedOut:
case VariantAnalysisRepoStatusDto.TimedOut:
return VariantAnalysisRepoStatus.TimedOut;
default:
assertNever(analysisStatus);

View File

@@ -4,20 +4,20 @@ import {
VariantAnalysisRepoStatus,
} from "../shared/variant-analysis";
import {
VariantAnalysisRepositoryTaskData,
VariantAnalysisRepoStatusData,
} from "./repo-task-data-types";
VariantAnalysisRepositoryTaskDto,
VariantAnalysisRepoStatusDto,
} from "./repo-tasks-dto";
export function mapRepoTaskToData(
export function mapRepoTaskToDto(
repoTask: VariantAnalysisRepositoryTask,
): VariantAnalysisRepositoryTaskData {
): VariantAnalysisRepositoryTaskDto {
return {
repository: {
id: repoTask.repository.id,
fullName: repoTask.repository.fullName,
private: repoTask.repository.private,
},
analysisStatus: mapRepoTaskAnalysisStatusToData(repoTask.analysisStatus),
analysisStatus: mapRepoTaskAnalysisStatusToDto(repoTask.analysisStatus),
resultCount: repoTask.resultCount,
artifactSizeInBytes: repoTask.artifactSizeInBytes,
failureMessage: repoTask.failureMessage,
@@ -27,22 +27,22 @@ export function mapRepoTaskToData(
};
}
function mapRepoTaskAnalysisStatusToData(
function mapRepoTaskAnalysisStatusToDto(
analysisStatus: VariantAnalysisRepoStatus,
): VariantAnalysisRepoStatusData {
): VariantAnalysisRepoStatusDto {
switch (analysisStatus) {
case VariantAnalysisRepoStatus.Pending:
return VariantAnalysisRepoStatusData.Pending;
return VariantAnalysisRepoStatusDto.Pending;
case VariantAnalysisRepoStatus.InProgress:
return VariantAnalysisRepoStatusData.InProgress;
return VariantAnalysisRepoStatusDto.InProgress;
case VariantAnalysisRepoStatus.Succeeded:
return VariantAnalysisRepoStatusData.Succeeded;
return VariantAnalysisRepoStatusDto.Succeeded;
case VariantAnalysisRepoStatus.Failed:
return VariantAnalysisRepoStatusData.Failed;
return VariantAnalysisRepoStatusDto.Failed;
case VariantAnalysisRepoStatus.Canceled:
return VariantAnalysisRepoStatusData.Canceled;
return VariantAnalysisRepoStatusDto.Canceled;
case VariantAnalysisRepoStatus.TimedOut:
return VariantAnalysisRepoStatusData.TimedOut;
return VariantAnalysisRepoStatusDto.TimedOut;
default:
assertNever(analysisStatus);
}

View File

@@ -1,6 +1,6 @@
export interface VariantAnalysisRepositoryTaskData {
repository: RepositoryData;
analysisStatus: VariantAnalysisRepoStatusData;
export interface VariantAnalysisRepositoryTaskDto {
repository: RepositoryDto;
analysisStatus: VariantAnalysisRepoStatusDto;
resultCount?: number;
artifactSizeInBytes?: number;
failureMessage?: string;
@@ -9,13 +9,13 @@ export interface VariantAnalysisRepositoryTaskData {
artifactUrl?: string;
}
interface RepositoryData {
interface RepositoryDto {
id: number;
fullName: string;
private: boolean;
}
export enum VariantAnalysisRepoStatusData {
export enum VariantAnalysisRepoStatusDto {
Pending = "pending",
InProgress = "inProgress",
Succeeded = "succeeded",

View File

@@ -1,8 +1,8 @@
import { outputJson, readJson } from "fs-extra";
import { join } from "path";
import { VariantAnalysisRepositoryTask } from "../shared/variant-analysis";
import { mapRepoTaskToData } from "./repo-task-to-data-mapper";
import { mapRepoTaskToDomain } from "./repo-task-to-domain-mapper";
import { mapRepoTaskToDto } from "./repo-tasks-dto-mapper";
import { mapRepoTaskToDomainModel } from "./repo-tasks-domain-mapper";
export const REPO_TASK_FILENAME = "repo_task.json";
@@ -10,7 +10,7 @@ export async function writeRepoTask(
storageDirectory: string,
repoTask: VariantAnalysisRepositoryTask,
): Promise<void> {
const repoTaskData = mapRepoTaskToData(repoTask);
const repoTaskData = mapRepoTaskToDto(repoTask);
await outputJson(join(storageDirectory, REPO_TASK_FILENAME), repoTaskData);
}
@@ -20,5 +20,5 @@ export async function readRepoTask(
const repoTaskData = await readJson(
join(storageDirectory, REPO_TASK_FILENAME),
);
return mapRepoTaskToDomain(repoTaskData);
return mapRepoTaskToDomainModel(repoTaskData);
}

View File

@@ -1,12 +0,0 @@
export interface VariantAnalysisScannedRepositoryStateData {
repositoryId: number;
downloadStatus: VariantAnalysisScannedRepositoryDownloadData;
downloadPercentage?: number;
}
export enum VariantAnalysisScannedRepositoryDownloadData {
Pending = "pending",
InProgress = "inProgress",
Succeeded = "succeeded",
Failed = "failed",
}

View File

@@ -1,36 +0,0 @@
import { assertNever } from "../../pure/helpers-pure";
import {
VariantAnalysisScannedRepositoryDownloadStatus,
VariantAnalysisScannedRepositoryState,
} from "../shared/variant-analysis";
import {
VariantAnalysisScannedRepositoryDownloadData,
VariantAnalysisScannedRepositoryStateData,
} from "./repo-states-data-types";
export function mapRepoStateToData(
repoState: VariantAnalysisScannedRepositoryState,
): VariantAnalysisScannedRepositoryStateData {
return {
repositoryId: repoState.repositoryId,
downloadStatus: processDownloadStatus(repoState.downloadStatus),
downloadPercentage: repoState.downloadPercentage,
};
}
function processDownloadStatus(
downloadedStatus: VariantAnalysisScannedRepositoryDownloadStatus,
) {
switch (downloadedStatus) {
case VariantAnalysisScannedRepositoryDownloadStatus.Pending:
return VariantAnalysisScannedRepositoryDownloadData.Pending;
case VariantAnalysisScannedRepositoryDownloadStatus.InProgress:
return VariantAnalysisScannedRepositoryDownloadData.InProgress;
case VariantAnalysisScannedRepositoryDownloadStatus.Succeeded:
return VariantAnalysisScannedRepositoryDownloadData.Succeeded;
case VariantAnalysisScannedRepositoryDownloadStatus.Failed:
return VariantAnalysisScannedRepositoryDownloadData.Failed;
default:
assertNever(downloadedStatus);
}
}

View File

@@ -1,36 +0,0 @@
import { assertNever } from "../../pure/helpers-pure";
import {
VariantAnalysisScannedRepositoryState,
VariantAnalysisScannedRepositoryDownloadStatus,
} from "../shared/variant-analysis";
import {
VariantAnalysisScannedRepositoryStateData,
VariantAnalysisScannedRepositoryDownloadData,
} from "./repo-states-data-types";
export function mapRepoStateToDomain(
repoState: VariantAnalysisScannedRepositoryStateData,
): VariantAnalysisScannedRepositoryState {
return {
repositoryId: repoState.repositoryId,
downloadStatus: processDownloadStatus(repoState.downloadStatus),
downloadPercentage: repoState.downloadPercentage,
};
}
function processDownloadStatus(
downloadedStatus: VariantAnalysisScannedRepositoryDownloadData,
) {
switch (downloadedStatus) {
case VariantAnalysisScannedRepositoryDownloadData.Pending:
return VariantAnalysisScannedRepositoryDownloadStatus.Pending;
case VariantAnalysisScannedRepositoryDownloadData.InProgress:
return VariantAnalysisScannedRepositoryDownloadStatus.InProgress;
case VariantAnalysisScannedRepositoryDownloadData.Succeeded:
return VariantAnalysisScannedRepositoryDownloadStatus.Succeeded;
case VariantAnalysisScannedRepositoryDownloadData.Failed:
return VariantAnalysisScannedRepositoryDownloadStatus.Failed;
default:
assertNever(downloadedStatus);
}
}

View File

@@ -71,7 +71,7 @@ import {
readRepoStates,
REPO_STATES_FILENAME,
writeRepoStates,
} from "./store/repo-states-store";
} from "./repo-states-store";
export class VariantAnalysisManager
extends DisposableObject

View File

@@ -17,7 +17,7 @@ import {
import { DisposableObject, DisposeHandler } from "../pure/disposable-object";
import { EventEmitter } from "vscode";
import { unzipFile } from "../pure/zip";
import { readRepoTask, writeRepoTask } from "./store/repo-task-store";
import { readRepoTask, writeRepoTask } from "./repo-tasks-store";
type CacheKey = `${number}/${string}`;

View File

@@ -0,0 +1,48 @@
import * as React from "react";
import { useCallback, useEffect } from "react";
import styled from "styled-components";
import { VSCodeDropdown, VSCodeOption } from "@vscode/webview-ui-toolkit/react";
import type { ModeledMethod } from "../../data-extensions-editor/modeled-method";
const Dropdown = styled(VSCodeDropdown)`
width: 100%;
`;
type Props = {
kinds: Array<ModeledMethod["kind"]>;
value: ModeledMethod["kind"] | undefined;
onChange: (value: ModeledMethod["kind"]) => void;
};
export const KindInput = ({ kinds, value, onChange }: Props) => {
const handleInput = useCallback(
(e: InputEvent) => {
const target = e.target as HTMLSelectElement;
onChange(target.value as ModeledMethod["kind"]);
},
[onChange],
);
useEffect(() => {
if (value === undefined && kinds.length > 0) {
onChange(kinds[0]);
}
if (value !== undefined && !kinds.includes(value)) {
onChange(kinds[0]);
}
}, [value, kinds, onChange]);
return (
<Dropdown value={value} onInput={handleInput}>
{kinds.map((kind) => (
<VSCodeOption key={kind} value={kind}>
{kind}
</VSCodeOption>
))}
</Dropdown>
);
};

View File

@@ -3,7 +3,6 @@ import {
VSCodeDataGridRow,
VSCodeDropdown,
VSCodeOption,
VSCodeTextField,
} from "@vscode/webview-ui-toolkit/react";
import * as React from "react";
import { useCallback, useMemo } from "react";
@@ -15,15 +14,13 @@ import {
ModeledMethod,
ModeledMethodType,
} from "../../data-extensions-editor/modeled-method";
import { KindInput } from "./KindInput";
import { extensiblePredicateDefinitions } from "../../data-extensions-editor/predicates";
const Dropdown = styled(VSCodeDropdown)`
width: 100%;
`;
const TextField = styled(VSCodeTextField)`
width: 100%;
`;
type SupportedUnsupportedSpanProps = {
supported: boolean;
};
@@ -107,17 +104,15 @@ export const MethodRow = ({
},
[onChange, externalApiUsage, modeledMethod],
);
const handleKindInput = useCallback(
(e: InputEvent) => {
const handleKindChange = useCallback(
(kind: string) => {
if (!modeledMethod) {
return;
}
const target = e.target as HTMLSelectElement;
onChange(externalApiUsage, {
...modeledMethod,
kind: target.value as ModeledMethod["kind"],
kind,
});
},
[onChange, externalApiUsage, modeledMethod],
@@ -130,6 +125,11 @@ export const MethodRow = ({
});
}, [externalApiUsage]);
const predicate =
modeledMethod?.type && modeledMethod.type !== "none"
? extensiblePredicateDefinitions[modeledMethod.type]
: undefined;
return (
<VSCodeDataGridRow>
<VSCodeDataGridCell gridColumn={1}>
@@ -195,10 +195,13 @@ export const MethodRow = ({
)}
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={7}>
{modeledMethod?.type &&
["source", "sink", "summary"].includes(modeledMethod?.type) && (
<TextField value={modeledMethod?.kind} onInput={handleKindInput} />
)}
{predicate?.supportedKinds && (
<KindInput
kinds={predicate.supportedKinds}
value={modeledMethod?.kind}
onChange={handleKindChange}
/>
)}
</VSCodeDataGridCell>
</VSCodeDataGridRow>
);

View File

@@ -149,14 +149,14 @@ describe("loadDataExtensionYaml", () => {
});
it("returns undefined if given a string", () => {
const data = loadDataExtensionYaml(`extensions:
expect(() =>
loadDataExtensionYaml(`extensions:
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data:
- ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","manual"]
`);
expect(data).toBeUndefined();
`),
).toThrow("Invalid data extension YAML: must be object");
});
});

View File

@@ -53,7 +53,7 @@ describe("Db panel UI commands", () => {
it.skip("should add new local db list", async () => {
// Add db list
jest.spyOn(window, "showQuickPick").mockResolvedValue({
kind: DbListKind.Local,
databaseKind: DbListKind.Local,
} as AddListQuickPickItem);
jest.spyOn(window, "showInputBox").mockResolvedValue("my-list-1");
await commandManager.execute(
@@ -73,7 +73,7 @@ describe("Db panel UI commands", () => {
it("should add new remote repository", async () => {
// Add db
jest.spyOn(window, "showQuickPick").mockResolvedValue({
kind: "repo",
remoteDatabaseKind: "repo",
} as RemoteDatabaseQuickPickItem);
jest.spyOn(window, "showInputBox").mockResolvedValue("owner1/repo1");
@@ -96,7 +96,7 @@ describe("Db panel UI commands", () => {
it("should add new remote owner", async () => {
// Add owner
jest.spyOn(window, "showQuickPick").mockResolvedValue({
kind: "owner",
remoteDatabaseKind: "owner",
} as RemoteDatabaseQuickPickItem);
jest.spyOn(window, "showInputBox").mockResolvedValue("owner1");

View File

@@ -48,7 +48,7 @@ import { mockedObject } from "../../utils/mocking.helpers";
import {
REPO_STATES_FILENAME,
writeRepoStates,
} from "../../../../src/variant-analysis/store/repo-states-store";
} from "../../../../src/variant-analysis/repo-states-store";
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);

View File

@@ -16,8 +16,6 @@ import {
} from "../global.helper";
import { createMockCommandManager } from "../../__mocks__/commandsMock";
jest.setTimeout(60_000);
/**
* Run various integration tests for databases
*/

View File

@@ -4,9 +4,6 @@ import { CodeQLCliServer } from "../../../src/cli";
import { tryGetQueryMetadata } from "../../../src/helpers";
import { getActivatedExtension } from "../global.helper";
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
describe("helpers (with CLI)", () => {
const baseDir = __dirname;

View File

@@ -6,6 +6,9 @@ const config: Config = {
...baseConfig,
runner: "<rootDir>/../jest-runner-installed-extensions.ts",
setupFilesAfterEnv: ["<rootDir>/jest.setup.ts"],
// CLI integration tests call into the CLI and execute queries, so these are expected to take a lot longer
// than the default 5 seconds.
testTimeout: 180_000, // 3 minutes
};
export default config;

View File

@@ -98,8 +98,6 @@ const db: messages.Dataset = {
workingSet: "default",
};
jest.setTimeout(60_000);
describeWithCodeQL()("using the legacy query server", () => {
const nullProgressReporter: ProgressReporter = {
report: () => {

View File

@@ -104,8 +104,6 @@ const nullProgressReporter: ProgressReporter = {
},
};
jest.setTimeout(20_000);
describeWithCodeQL()("using the new query server", () => {
let qs: qsClient.QueryServerClient;
let cliServer: cli.CodeQLCliServer;

View File

@@ -12,9 +12,6 @@ import {
import { mockedQuickPickItem } from "../utils/mocking.helpers";
import { getActivatedExtension } from "../global.helper";
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
describe("Packaging commands", () => {
let cli: CodeQLCliServer;
const progress = jest.fn();

View File

@@ -27,8 +27,6 @@ import { QueryResultType } from "../../../src/pure/new-messages";
import { createVSCodeCommandManager } from "../../../src/common/vscode/commands";
import { AllCommands, QueryServerCommands } from "../../../src/common/commands";
jest.setTimeout(20_000);
/**
* Integration tests for queries
*/

View File

@@ -14,8 +14,6 @@ import { KeyType } from "../../../src/contextual/keyType";
import { faker } from "@faker-js/faker";
import { getActivatedExtension } from "../global.helper";
jest.setTimeout(60_000);
/**
* Perform proper integration tests by running the CLI
*/

View File

@@ -20,8 +20,7 @@ import {
} from "../../../src/local-databases";
import * as databaseFetcher from "../../../src/databaseFetcher";
import { createMockDB } from "../../factories/databases/databases";
jest.setTimeout(40_000);
import { asError } from "../../../src/pure/helpers-pure";
describe("SkeletonQueryWizard", () => {
let mockCli: CodeQLCliServer;
@@ -306,8 +305,15 @@ describe("SkeletonQueryWizard", () => {
describe("findDatabaseItemByNwo", () => {
describe("when the item exists", () => {
it("should return the database item", async () => {
const mockDbItem = createMockDB(dir);
const mockDbItem2 = createMockDB(dir);
const mockDbItem = createMockDB(dir, {
language: "ruby",
dateAdded: 123,
} as FullDatabaseOptions);
const mockDbItem2 = createMockDB(dir, {
language: "javascript",
} as FullDatabaseOptions);
jest.spyOn(mockDbItem, "name", "get").mockReturnValue("mock-name");
const databaseItem = await wizard.findDatabaseItemByNwo(
mockDbItem.language,
@@ -315,8 +321,40 @@ describe("SkeletonQueryWizard", () => {
[mockDbItem, mockDbItem2],
);
expect(databaseItem!.language).toEqual(mockDbItem.language);
expect(databaseItem!.name).toEqual(mockDbItem.name);
expect(JSON.stringify(databaseItem)).toEqual(
JSON.stringify(mockDbItem),
);
});
it("should ignore databases with errors", async () => {
const mockDbItem = createMockDB(dir, {
language: "ruby",
dateAdded: 123,
} as FullDatabaseOptions);
const mockDbItem2 = createMockDB(dir, {
language: "javascript",
} as FullDatabaseOptions);
const mockDbItem3 = createMockDB(dir, {
language: "ruby",
dateAdded: 345,
} as FullDatabaseOptions);
jest.spyOn(mockDbItem, "name", "get").mockReturnValue("mock-name");
jest.spyOn(mockDbItem3, "name", "get").mockReturnValue(mockDbItem.name);
jest
.spyOn(mockDbItem, "error", "get")
.mockReturnValue(asError("database go boom!"));
const databaseItem = await wizard.findDatabaseItemByNwo(
mockDbItem.language,
mockDbItem.name,
[mockDbItem, mockDbItem2, mockDbItem3],
);
expect(JSON.stringify(databaseItem)).toEqual(
JSON.stringify(mockDbItem3),
);
});
});
@@ -353,6 +391,32 @@ describe("SkeletonQueryWizard", () => {
expect(databaseItem).toEqual(mockDbItem);
});
it("should ignore databases with errors", async () => {
const mockDbItem = createMockDB(dir, {
language: "ruby",
} as FullDatabaseOptions);
const mockDbItem2 = createMockDB(dir, {
language: "javascript",
} as FullDatabaseOptions);
const mockDbItem3 = createMockDB(dir, {
language: "ruby",
} as FullDatabaseOptions);
jest
.spyOn(mockDbItem, "error", "get")
.mockReturnValue(asError("database go boom!"));
const databaseItem = await wizard.findDatabaseItemByLanguage("ruby", [
mockDbItem,
mockDbItem2,
mockDbItem3,
]);
expect(JSON.stringify(databaseItem)).toEqual(
JSON.stringify(mockDbItem3),
);
});
});
describe("when the item doesn't exist", () => {

View File

@@ -5,8 +5,6 @@ import { readFile, writeFile, ensureDir, copy } from "fs-extra";
import { createVSCodeCommandManager } from "../../../src/common/vscode/commands";
import { AllCommands } from "../../../src/common/commands";
jest.setTimeout(20_000);
/**
* Integration tests for queries
*/

View File

@@ -29,9 +29,6 @@ import { QueryLanguage } from "../../../../src/common/query-language";
import { readBundledPack } from "../../utils/bundled-pack-helpers";
import { load } from "js-yaml";
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
describe("Variant Analysis Manager", () => {
let cli: CodeQLCliServer;
let cancellationTokenSource: CancellationTokenSource;

View File

@@ -15,8 +15,6 @@ import { getActivatedExtension } from "../../global.helper";
import { createVSCodeCommandManager } from "../../../../src/common/vscode/commands";
import { AllCommands } from "../../../../src/common/commands";
jest.setTimeout(30_000);
const mockServer = new MockGitHubApiServer();
beforeAll(() => mockServer.startServer());
afterEach(() => mockServer.unloadScenario());

View File

@@ -1,4 +1,8 @@
import { QuickPickItem, window } from "vscode";
import { CancellationTokenSource, QuickPickItem, window } from "vscode";
import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import { outputFile, readFile } from "fs-extra";
import { join } from "path";
import { dir } from "tmp-promise";
import { pickExtensionPackModelFile } from "../../../../src/data-extensions-editor/extension-pack-picker";
import { QlpacksInfo, ResolveExtensionsResult } from "../../../../src/cli";
@@ -21,14 +25,32 @@ describe("pickExtensionPackModelFile", () => {
],
},
};
const databaseItem = {
name: "github/vscode-codeql",
};
const cancellationTokenSource = new CancellationTokenSource();
const token = cancellationTokenSource.token;
const progress = jest.fn();
let showQuickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
let showInputBoxSpy: jest.SpiedFunction<typeof window.showInputBox>;
let showAndLogErrorMessageSpy: jest.SpiedFunction<
typeof helpers.showAndLogErrorMessage
>;
beforeEach(() => {
showQuickPickSpy = jest
.spyOn(window, "showQuickPick")
.mockRejectedValue(new Error("Unexpected call to showQuickPick"));
showInputBoxSpy = jest
.spyOn(window, "showInputBox")
.mockRejectedValue(new Error("Unexpected call to showInputBox"));
showAndLogErrorMessageSpy = jest
.spyOn(helpers, "showAndLogErrorMessage")
.mockImplementation((msg) => {
throw new Error(`Unexpected call to showAndLogErrorMessage: ${msg}`);
});
});
it("allows choosing an existing extension pack and model file", async () => {
@@ -43,9 +65,14 @@ describe("pickExtensionPackModelFile", () => {
file: "/a/b/c/my-extension-pack/models/model.yml",
} as QuickPickItem);
expect(await pickExtensionPackModelFile(cliServer, progress)).toEqual(
"/a/b/c/my-extension-pack/models/model.yml",
);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual("/a/b/c/my-extension-pack/models/model.yml");
expect(showQuickPickSpy).toHaveBeenCalledTimes(2);
expect(showQuickPickSpy).toHaveBeenCalledWith(
[
@@ -57,10 +84,15 @@ describe("pickExtensionPackModelFile", () => {
label: "another-extension-pack",
extensionPack: "another-extension-pack",
},
{
label: expect.stringMatching(/create/i),
extensionPack: null,
},
],
{
title: expect.any(String),
},
token,
);
expect(showQuickPickSpy).toHaveBeenCalledWith(
[
@@ -68,10 +100,15 @@ describe("pickExtensionPackModelFile", () => {
label: "models/model.yml",
file: "/a/b/c/my-extension-pack/models/model.yml",
},
{
label: expect.stringMatching(/create/i),
file: null,
},
],
{
title: expect.any(String),
},
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
@@ -82,39 +119,235 @@ describe("pickExtensionPackModelFile", () => {
);
});
it("allows choosing an existing extension pack and creating a new model file", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
...qlPacks,
"my-extension-pack": [tmpDir.path],
},
{
models: extensions.models,
data: {
[tmpDir.path]: [
{
file: join(tmpDir.path, "models/model.yml"),
index: 0,
predicate: "sinkModel",
},
],
},
},
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce({
label: "create",
file: null,
} as QuickPickItem);
showInputBoxSpy.mockResolvedValue("models/my-model.yml");
await outputFile(
join(tmpDir.path, "codeql-pack.yml"),
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
}),
);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(join(tmpDir.path, "models/my-model.yml"));
expect(showQuickPickSpy).toHaveBeenCalledTimes(2);
expect(showQuickPickSpy).toHaveBeenCalledWith(
[
{
label: "my-extension-pack",
extensionPack: "my-extension-pack",
},
{
label: "another-extension-pack",
extensionPack: "another-extension-pack",
},
{
label: expect.stringMatching(/create/i),
extensionPack: null,
},
],
{
title: expect.any(String),
},
token,
);
expect(showQuickPickSpy).toHaveBeenCalledWith(
[
{
label: "models/model.yml",
file: join(tmpDir.path, "models/model.yml"),
},
{
label: expect.stringMatching(/create/i),
file: null,
},
],
{
title: expect.any(String),
},
token,
);
expect(showInputBoxSpy).toHaveBeenCalledWith(
{
title: expect.any(String),
value: "models/github.vscode-codeql.model.yml",
validateInput: expect.any(Function),
},
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
expect(cliServer.resolveExtensions).toHaveBeenCalledTimes(1);
expect(cliServer.resolveExtensions).toHaveBeenCalledWith(tmpDir.path, []);
});
it("allows cancelling the extension pack prompt", async () => {
const cliServer = mockCliServer(qlPacks, extensions);
showQuickPickSpy.mockResolvedValueOnce(undefined);
expect(await pickExtensionPackModelFile(cliServer, progress)).toEqual(
undefined,
);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).not.toHaveBeenCalled();
});
it("does not show any options when there are no extension packs", async () => {
it("allows user to create an extension pack when there are no extension packs", async () => {
const cliServer = mockCliServer({}, { models: [], data: {} });
const tmpDir = await dir({
unsafeCleanup: true,
});
showQuickPickSpy.mockResolvedValueOnce({
label: "codeql-custom-queries-java",
path: tmpDir.path,
} as QuickPickItem);
showInputBoxSpy.mockResolvedValueOnce("my-extension-pack");
showInputBoxSpy.mockResolvedValue("models/my-model.yml");
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(join(tmpDir.path, "my-extension-pack", "models", "my-model.yml"));
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).toHaveBeenCalledTimes(2);
expect(showInputBoxSpy).toHaveBeenCalledWith(
{
title: expect.stringMatching(/extension pack/i),
prompt: expect.stringMatching(/extension pack/i),
placeHolder: expect.stringMatching(/github\/vscode-codeql-extensions/),
validateInput: expect.any(Function),
},
token,
);
expect(showInputBoxSpy).toHaveBeenCalledWith(
{
title: expect.stringMatching(/model file/),
value: "models/github.vscode-codeql.model.yml",
validateInput: expect.any(Function),
},
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
expect(
loadYaml(
await readFile(
join(tmpDir.path, "my-extension-pack", "codeql-pack.yml"),
"utf8",
),
),
).toEqual({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
});
});
it("allows cancelling the workspace folder selection", async () => {
const cliServer = mockCliServer({}, { models: [], data: {} });
showQuickPickSpy.mockResolvedValueOnce(undefined);
expect(await pickExtensionPackModelFile(cliServer, progress)).toEqual(
undefined,
);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showQuickPickSpy).toHaveBeenCalledWith([], {
title: expect.any(String),
});
expect(showInputBoxSpy).toHaveBeenCalledTimes(0);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).not.toHaveBeenCalled();
});
it("allows cancelling the extension pack name input", async () => {
const cliServer = mockCliServer({}, { models: [], data: {} });
showQuickPickSpy.mockResolvedValueOnce({
label: "codeql-custom-queries-java",
path: "/a/b/c",
} as QuickPickItem);
showInputBoxSpy.mockResolvedValueOnce(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).not.toHaveBeenCalled();
});
it("shows an error when an extension pack resolves to more than 1 location", async () => {
const showAndLogErrorMessageSpy = jest.spyOn(
helpers,
"showAndLogErrorMessage",
);
showAndLogErrorMessageSpy.mockResolvedValue(undefined);
const cliServer = mockCliServer(
{
@@ -131,9 +364,14 @@ describe("pickExtensionPackModelFile", () => {
extensionPack: "my-extension-pack",
} as QuickPickItem);
expect(await pickExtensionPackModelFile(cliServer, progress)).toEqual(
undefined,
);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showAndLogErrorMessageSpy).toHaveBeenCalledTimes(1);
expect(showAndLogErrorMessageSpy).toHaveBeenCalledWith(
expect.stringMatching(/could not be resolved to a single location/),
@@ -153,47 +391,468 @@ describe("pickExtensionPackModelFile", () => {
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
expect(await pickExtensionPackModelFile(cliServer, progress)).toEqual(
undefined,
);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("does not show any options when there are no model files", async () => {
const cliServer = mockCliServer(qlPacks, { models: [], data: {} });
it("shows create input box when there are no model files", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
await outputFile(
join(tmpDir.path, "codeql-pack.yml"),
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
}),
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showInputBoxSpy.mockResolvedValue("models/my-model.yml");
expect(await pickExtensionPackModelFile(cliServer, progress)).toEqual(
undefined,
);
expect(showQuickPickSpy).toHaveBeenCalledTimes(2);
expect(showQuickPickSpy).toHaveBeenCalledWith(
[
{
label: "my-extension-pack",
extensionPack: "my-extension-pack",
},
{
label: "another-extension-pack",
extensionPack: "another-extension-pack",
},
],
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(join(tmpDir.path, "models/my-model.yml"));
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).toHaveBeenCalledWith(
{
title: expect.any(String),
value: "models/github.vscode-codeql.model.yml",
validateInput: expect.any(Function),
},
token,
);
expect(showQuickPickSpy).toHaveBeenCalledWith([], {
title: expect.any(String),
});
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("shows an error when there is no pack YAML file", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showAndLogErrorMessageSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).not.toHaveBeenCalled();
expect(showAndLogErrorMessageSpy).toHaveBeenCalledTimes(1);
expect(showAndLogErrorMessageSpy).toHaveBeenCalledWith(
expect.stringMatching(/codeql-pack\.yml/),
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("shows an error when the pack YAML file is invalid", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
await outputFile(join(tmpDir.path, "codeql-pack.yml"), dumpYaml("java"));
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showAndLogErrorMessageSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).not.toHaveBeenCalled();
expect(showAndLogErrorMessageSpy).toHaveBeenCalledTimes(1);
expect(showAndLogErrorMessageSpy).toHaveBeenCalledWith(
expect.stringMatching(/Could not parse/),
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("shows an error when the pack YAML does not contain dataExtensions", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
await outputFile(
join(tmpDir.path, "codeql-pack.yml"),
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
}),
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showAndLogErrorMessageSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).not.toHaveBeenCalled();
expect(showAndLogErrorMessageSpy).toHaveBeenCalledTimes(1);
expect(showAndLogErrorMessageSpy).toHaveBeenCalledWith(
expect.stringMatching(/Expected 'dataExtensions' to be/),
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("shows an error when the pack YAML dataExtensions is invalid", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
await outputFile(
join(tmpDir.path, "codeql-pack.yml"),
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: {
"codeql/java-all": "invalid",
},
}),
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showAndLogErrorMessageSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).not.toHaveBeenCalled();
expect(showAndLogErrorMessageSpy).toHaveBeenCalledTimes(1);
expect(showAndLogErrorMessageSpy).toHaveBeenCalledWith(
expect.stringMatching(/Expected 'dataExtensions' to be/),
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("allows cancelling the new file input box", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
await outputFile(
join(tmpDir.path, "codeql-pack.yml"),
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
}),
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showInputBoxSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
});
it("validates the pack name input", async () => {
const cliServer = mockCliServer({}, { models: [], data: {} });
showQuickPickSpy.mockResolvedValueOnce({
label: "a",
path: "/a/b/c",
} as QuickPickItem);
showInputBoxSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
const validateFile = showInputBoxSpy.mock.calls[0][0]?.validateInput;
expect(validateFile).toBeDefined();
if (!validateFile) {
return;
}
expect(await validateFile("")).toEqual("Pack name must not be empty");
expect(await validateFile("a".repeat(129))).toEqual(
"Pack name must be no longer than 128 characters",
);
expect(await validateFile("github/vscode-codeql/extensions")).toEqual(
"Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens",
);
expect(await validateFile("VSCODE")).toEqual(
"Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens",
);
expect(await validateFile("github/vscode-codeql-")).toEqual(
"Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens",
);
expect(
await validateFile("github/vscode-codeql-extensions"),
).toBeUndefined();
expect(await validateFile("vscode-codeql-extensions")).toBeUndefined();
});
it("validates the file input", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
const qlpackPath = join(tmpDir.path, "codeql-pack.yml");
await outputFile(
qlpackPath,
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml", "data/**/*.yml"],
}),
);
await outputFile(
join(tmpDir.path, "models", "model.yml"),
dumpYaml({
extensions: [],
}),
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showInputBoxSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
const validateFile = showInputBoxSpy.mock.calls[0][0]?.validateInput;
expect(validateFile).toBeDefined();
if (!validateFile) {
return;
}
expect(await validateFile("")).toEqual("File name must not be empty");
expect(await validateFile("models/model.yml")).toEqual(
"File already exists",
);
expect(await validateFile("../model.yml")).toEqual(
"File must be in the extension pack",
);
expect(await validateFile("/home/user/model.yml")).toEqual(
"File must be in the extension pack",
);
expect(await validateFile("model.yml")).toEqual(
`File must match one of the patterns in 'dataExtensions' in ${qlpackPath}`,
);
expect(await validateFile("models/model.yaml")).toEqual(
`File must match one of the patterns in 'dataExtensions' in ${qlpackPath}`,
);
expect(await validateFile("models/my-model.yml")).toBeUndefined();
expect(await validateFile("models/nested/model.yml")).toBeUndefined();
expect(await validateFile("data/model.yml")).toBeUndefined();
});
it("allows the dataExtensions to be a string", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer(
{
"my-extension-pack": [tmpDir.path],
},
{ models: [], data: {} },
);
const qlpackPath = join(tmpDir.path, "codeql-pack.yml");
await outputFile(
qlpackPath,
dumpYaml({
name: "my-extension-pack",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: "models/**/*.yml",
}),
);
await outputFile(
join(tmpDir.path, "models", "model.yml"),
dumpYaml({
extensions: [],
}),
);
showQuickPickSpy.mockResolvedValueOnce({
label: "my-extension-pack",
extensionPack: "my-extension-pack",
} as QuickPickItem);
showQuickPickSpy.mockResolvedValueOnce(undefined);
showInputBoxSpy.mockResolvedValue(undefined);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
progress,
token,
),
).toEqual(undefined);
const validateFile = showInputBoxSpy.mock.calls[0][0]?.validateInput;
expect(validateFile).toBeDefined();
if (!validateFile) {
return;
}
expect(await validateFile("models/my-model.yml")).toBeUndefined();
});
});
function mockCliServer(

View File

@@ -8,8 +8,12 @@ import { DatabaseKind } from "../../../../src/local-databases";
import * as queryResolver from "../../../../src/contextual/queryResolver";
import { file } from "tmp-promise";
import { QueryResultType } from "../../../../src/pure/new-messages";
import { readFile } from "fs-extra";
import { readdir, readFile } from "fs-extra";
import { load } from "js-yaml";
import { dirname, join } from "path";
import { fetchExternalApisQuery } from "../../../../src/data-extensions-editor/queries/java";
import * as helpers from "../../../../src/helpers";
import { RedactableError } from "../../../../src/pure/errors";
function createMockUri(path = "/a/b/c/foo"): Uri {
return {
@@ -39,11 +43,6 @@ describe("runQuery", () => {
resolveQlpacks: jest.fn().mockResolvedValue({
"my/java-extensions": "/a/b/c/",
}),
resolveQueriesInSuite: jest
.fn()
.mockResolvedValue([
"/home/github/codeql/java/ql/src/Telemetry/FetchExternalAPIs.ql",
]),
},
queryRunner: {
createQueryRun: jest.fn().mockReturnValue({
@@ -56,7 +55,6 @@ describe("runQuery", () => {
}),
logger: createMockLogger(),
},
logger: createMockLogger(),
databaseItem: {
databaseUri: createMockUri("/a/b/c/src.zip"),
contents: {
@@ -77,37 +75,12 @@ describe("runQuery", () => {
expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
expect(options.cliServer.resolveQueriesInSuite).toHaveBeenCalledWith(
expect.anything(),
[],
);
const suiteFile = options.cliServer.resolveQueriesInSuite.mock.calls[0][0];
const suiteFileContents = await readFile(suiteFile, "utf8");
const suiteYaml = load(suiteFileContents);
expect(suiteYaml).toEqual([
{
from: "codeql/java-all",
queries: ".",
include: {
id: "java/telemetry/fetch-external-apis",
},
},
{
from: "codeql/java-queries",
queries: ".",
include: {
id: "java/telemetry/fetch-external-apis",
},
},
]);
expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(
"/a/b/c/src.zip",
{
queryPath:
"/home/github/codeql/java/ql/src/Telemetry/FetchExternalAPIs.ql",
queryPath: expect.stringMatching(/FetchExternalApis\.ql/),
quickEvalPosition: undefined,
},
false,
@@ -117,6 +90,40 @@ describe("runQuery", () => {
undefined,
undefined,
);
const queryPath =
options.queryRunner.createQueryRun.mock.calls[0][1].queryPath;
const queryDirectory = dirname(queryPath);
const queryFiles = await readdir(queryDirectory);
expect(queryFiles.sort()).toEqual(
["codeql-pack.yml", "FetchExternalApis.ql", "ExternalApi.qll"].sort(),
);
const suiteFileContents = await readFile(
join(queryDirectory, "codeql-pack.yml"),
"utf8",
);
const suiteYaml = load(suiteFileContents);
expect(suiteYaml).toEqual({
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
"codeql/java-all": "*",
},
});
expect(
await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"),
).toEqual(fetchExternalApisQuery.mainQuery);
for (const [filename, contents] of Object.entries(
fetchExternalApisQuery.dependencies ?? {},
)) {
expect(await readFile(join(queryDirectory, filename), "utf8")).toEqual(
contents,
);
}
});
});
@@ -127,17 +134,27 @@ describe("readQueryResults", () => {
bqrsDecode: jest.fn(),
},
bqrsPath: "/tmp/results.bqrs",
logger: createMockLogger(),
};
let showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction<
typeof helpers.showAndLogExceptionWithTelemetry
>;
beforeEach(() => {
showAndLogExceptionWithTelemetrySpy = jest.spyOn(
helpers,
"showAndLogExceptionWithTelemetry",
);
});
it("returns undefined when there are no results", async () => {
options.cliServer.bqrsInfo.mockResolvedValue({
"result-sets": [],
});
expect(await readQueryResults(options)).toBeUndefined();
expect(options.logger.log).toHaveBeenCalledWith(
expect.stringMatching(/Expected exactly one result set/),
expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith(
expect.any(RedactableError),
);
});
@@ -166,8 +183,8 @@ describe("readQueryResults", () => {
});
expect(await readQueryResults(options)).toBeUndefined();
expect(options.logger.log).toHaveBeenCalledWith(
expect.stringMatching(/Expected exactly one result set/),
expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith(
expect.any(RedactableError),
);
});

View File

@@ -7,7 +7,11 @@
"completed": true,
"variantAnalysis": {
"id": 98574321397,
"controllerRepoId": 128321,
"controllerRepo": {
"id": 128321,
"fullName": "github/codeql",
"private": false
},
"query": {
"name": "Variant Analysis Integration Test 1",
"filePath": "PLACEHOLDER/q2.ql",
@@ -30,7 +34,11 @@
"completed": true,
"variantAnalysis": {
"id": 98574321397,
"controllerRepoId": 128321,
"controllerRepo": {
"id": 128321,
"fullName": "github/codeql",
"private": false
},
"query": {
"name": "Variant Analysis Integration Test 2",
"filePath": "PLACEHOLDER/q2.ql",

View File

@@ -273,6 +273,13 @@ describe("helpers", () => {
class MockEnvironmentVariableCollection
implements EnvironmentVariableCollection
{
[Symbol.iterator](): Iterator<
[variable: string, mutator: EnvironmentVariableMutator],
any,
undefined
> {
throw new Error("Method not implemented.");
}
persistent = false;
replace(_variable: string, _value: string): void {
throw new Error("Method not implemented.");

View File

@@ -132,7 +132,10 @@ describe("Variant Analyses and QueryHistoryManager", () => {
await qhm.readQueryHistory();
// Remove the first variant analysis
await qhm.handleRemoveHistoryItem(qhm.treeDataProvider.allHistory[0]);
await qhm.handleRemoveHistoryItem(
qhm.treeDataProvider.allHistory[0],
undefined,
);
// Add it back to the history
qhm.addQuery(rawQueryHistory[0]);