Merge remote-tracking branch 'origin/main' into dbartol/debug-adapter

This commit is contained in:
Dave Bartolomeo
2023-04-13 09:54:40 -04:00
13 changed files with 1065 additions and 152 deletions

View File

@@ -2,8 +2,12 @@
## [UNRELEASED]
## 1.8.2 - 12 April 2023
- Fix bug where users could end up with the managed CodeQL CLI getting uninstalled during upgrades and not reinstalled. [#2294](https://github.com/github/vscode-codeql/pull/2294)
- Fix bug that was causing code flows to not get updated when switching between results. [#2288](https://github.com/github/vscode-codeql/pull/2288)
- Restart the CodeQL language server whenever the _CodeQL: Restart Query Server_ command is invoked. This avoids bugs where the CLI version changes to support new language features, but the language server is not updated. [#2238](https://github.com/github/vscode-codeql/pull/2238)
- Avoid requiring a manual restart of the query server when the [external CLI config file](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file) changes. [#2289](https://github.com/github/vscode-codeql/pull/2289)
## 1.8.1 - 23 March 2023

View File

@@ -1,12 +1,12 @@
{
"name": "vscode-codeql",
"version": "1.8.2",
"version": "1.8.3",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "vscode-codeql",
"version": "1.8.2",
"version": "1.8.3",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {

View File

@@ -4,7 +4,7 @@
"description": "CodeQL for Visual Studio Code",
"author": "GitHub",
"private": true,
"version": "1.8.2",
"version": "1.8.3",
"publisher": "GitHub",
"license": "MIT",
"icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -427,6 +427,10 @@
"command": "codeQL.quickQuery",
"title": "CodeQL: Quick Query"
},
{
"command": "codeQL.createSkeletonQuery",
"title": "CodeQL: Create Query"
},
{
"command": "codeQL.openDocumentation",
"title": "CodeQL: Open Documentation"

View File

@@ -81,6 +81,7 @@ export type BaseCommands = {
"codeQL.restartQueryServer": () => Promise<void>;
"codeQL.restartQueryServerOnConfigChange": () => Promise<void>;
"codeQL.restartLegacyQueryServerOnConfigChange": () => Promise<void>;
"codeQL.restartQueryServerOnExternalConfigChange": () => Promise<void>;
};
// Commands used when working with queries in the editor
@@ -109,6 +110,7 @@ export type LocalQueryCommands = {
"codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
"codeQL.quickQuery": () => Promise<void>;
"codeQL.getCurrentQuery": () => Promise<string>;
"codeQL.createSkeletonQuery": () => Promise<void>;
};
// Debugger commands

View File

@@ -1,7 +1,7 @@
import fetch, { Response } from "node-fetch";
import { zip } from "zip-a-folder";
import { Open } from "unzipper";
import { Uri, CancellationToken, window } from "vscode";
import { Uri, CancellationToken, window, InputBoxOptions } from "vscode";
import { CodeQLCliServer } from "./cli";
import {
ensureDir,
@@ -78,6 +78,10 @@ export async function promptImportInternetDatabase(
*
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param credentials the credentials to use to authenticate with GitHub
* @param progress the progress callback
* @param token the cancellation token
* @param cli the CodeQL CLI server
*/
export async function promptImportGithubDatabase(
commandManager: AppCommandManager,
@@ -88,21 +92,78 @@ export async function promptImportGithubDatabase(
token: CancellationToken,
cli?: CodeQLCliServer,
): Promise<DatabaseItem | undefined> {
progress({
message: "Choose repository",
step: 1,
maxStep: 2,
});
const githubRepo = await window.showInputBox({
title:
'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
placeHolder: "https://github.com/<owner>/<repo> or <owner>/<repo>",
ignoreFocusOut: true,
});
const githubRepo = await askForGitHubRepo(progress);
if (!githubRepo) {
return;
}
const databaseItem = await downloadGitHubDatabase(
githubRepo,
databaseManager,
storagePath,
credentials,
progress,
token,
cli,
);
if (databaseItem) {
await commandManager.execute("codeQLDatabases.focus");
void showAndLogInformationMessage(
"Database downloaded and imported successfully.",
);
return databaseItem;
}
return;
}
export async function askForGitHubRepo(
progress?: ProgressCallback,
suggestedValue?: string,
): Promise<string | undefined> {
progress?.({
message: "Choose repository",
step: 1,
maxStep: 2,
});
const options: InputBoxOptions = {
title:
'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
placeHolder: "https://github.com/<owner>/<repo> or <owner>/<repo>",
ignoreFocusOut: true,
};
if (suggestedValue) {
options.value = suggestedValue;
}
return await window.showInputBox(options);
}
/**
* Downloads a database from GitHub
*
* @param githubRepo the GitHub repository to download the database from
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param credentials the credentials to use to authenticate with GitHub
* @param progress the progress callback
* @param token the cancellation token
* @param cli the CodeQL CLI server
* @param language the language to download. If undefined, the user will be prompted to choose a language.
**/
export async function downloadGitHubDatabase(
githubRepo: string,
databaseManager: DatabaseManager,
storagePath: string,
credentials: Credentials | undefined,
progress: ProgressCallback,
token: CancellationToken,
cli?: CodeQLCliServer,
language?: string,
): Promise<DatabaseItem | undefined> {
const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo;
if (!isValidGitHubNwo(nwo)) {
throw new Error(`Invalid GitHub repository: ${githubRepo}`);
@@ -112,7 +173,12 @@ export async function promptImportGithubDatabase(
? await credentials.getOctokit()
: new Octokit.Octokit({ retry });
const result = await convertGithubNwoToDatabaseUrl(nwo, octokit, progress);
const result = await convertGithubNwoToDatabaseUrl(
nwo,
octokit,
progress,
language,
);
if (!result) {
return;
}
@@ -130,7 +196,7 @@ export async function promptImportGithubDatabase(
* We only need the actual token string.
*/
const octokitToken = ((await octokit.auth()) as { token: string })?.token;
const item = await databaseArchiveFetcher(
return await databaseArchiveFetcher(
databaseUrl,
{
Accept: "application/zip",
@@ -143,14 +209,6 @@ export async function promptImportGithubDatabase(
token,
cli,
);
if (item) {
await commandManager.execute("codeQLDatabases.focus");
void showAndLogInformationMessage(
"Database downloaded and imported successfully.",
);
return item;
}
return;
}
/**
@@ -450,6 +508,7 @@ export async function convertGithubNwoToDatabaseUrl(
nwo: string,
octokit: Octokit.Octokit,
progress: ProgressCallback,
language?: string,
): Promise<
| {
databaseUrl: string;
@@ -468,9 +527,11 @@ export async function convertGithubNwoToDatabaseUrl(
const languages = response.data.map((db: any) => db.language);
const language = await promptForLanguage(languages, progress);
if (!language) {
return;
if (!language || !languages.includes(language)) {
language = await promptForLanguage(languages, progress);
if (!language) {
return;
}
}
return {
@@ -484,7 +545,7 @@ export async function convertGithubNwoToDatabaseUrl(
}
}
async function promptForLanguage(
export async function promptForLanguage(
languages: string[],
progress: ProgressCallback,
): Promise<string | undefined> {

View File

@@ -13,12 +13,13 @@ import {
workspace,
} from "vscode";
import { LanguageClient } from "vscode-languageclient/node";
import { arch, platform } from "os";
import { arch, platform, homedir } from "os";
import { ensureDir } from "fs-extra";
import { join } from "path";
import { dirSync } from "tmp-promise";
import { testExplorerExtensionId, TestHub } from "vscode-test-adapter-api";
import { lt, parse } from "semver";
import { watch } from "chokidar";
import { AstViewer } from "./astViewer";
import {
@@ -196,6 +197,7 @@ function getCommands(
"codeQL.restartQueryServer": restartQueryServer,
"codeQL.restartQueryServerOnConfigChange": restartQueryServer,
"codeQL.restartLegacyQueryServerOnConfigChange": restartQueryServer,
"codeQL.restartQueryServerOnExternalConfigChange": restartQueryServer,
"codeQL.copyVersion": async () => {
const text = `CodeQL extension version: ${
extension?.packageJSON.version
@@ -674,6 +676,7 @@ async function activateWithInstalledDistribution(
extLogger,
);
ctx.subscriptions.push(cliServer);
watchExternalConfigFile(app, ctx);
const statusBar = new CodeQlStatusBarHandler(
cliServer,
@@ -1028,6 +1031,34 @@ async function activateWithInstalledDistribution(
};
}
/**
* Handle changes to the external config file. This is used to restart the query server
* when the user changes options.
* See https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file
*/
function watchExternalConfigFile(app: ExtensionApp, ctx: ExtensionContext) {
const home = homedir();
if (home) {
const configPath = join(home, ".config", "codeql", "config");
const configWatcher = watch(configPath, {
// These options avoid firing the event twice.
persistent: true,
ignoreInitial: true,
awaitWriteFinish: true,
});
configWatcher.on("all", async () => {
await app.commands.execute(
"codeQL.restartQueryServerOnExternalConfigChange",
);
});
ctx.subscriptions.push({
dispose: () => {
void configWatcher.close();
},
});
}
}
async function showResultsForComparison(
compareView: CompareView,
from: CompletedLocalQueryInfo,

View File

@@ -9,7 +9,7 @@ import {
workspace,
} from "vscode";
import { BaseLogger, extLogger, Logger, TeeLogger } from "./common";
import { MAX_QUERIES } from "./config";
import { isCanary, MAX_QUERIES } from "./config";
import { gatherQlFiles } from "./pure/files";
import { basename } from "path";
import {
@@ -54,6 +54,7 @@ import { App } from "./common/app";
import { DisposableObject } from "./pure/disposable-object";
import { QueryResultType } from "./pure/new-messages";
import { redactableError } from "./pure/errors";
import { SkeletonQueryWizard } from "./skeleton-query-wizard";
interface DatabaseQuickPickItem extends QuickPickItem {
databaseItem: DatabaseItem;
@@ -266,6 +267,7 @@ export class LocalQueries extends DisposableObject {
// sure.
return this.getCurrentQuery(true);
},
"codeQL.createSkeletonQuery": this.createSkeletonQuery.bind(this),
};
}
@@ -421,6 +423,26 @@ export class LocalQueries extends DisposableObject {
return validateQueryUri(editor.document.uri, allowLibraryFiles);
}
private async createSkeletonQuery(): Promise<void> {
await withProgress(
async (progress: ProgressCallback, token: CancellationToken) => {
const credentials = isCanary() ? this.app.credentials : undefined;
const skeletonQueryWizard = new SkeletonQueryWizard(
this.cliServer,
progress,
credentials,
extLogger,
this.databaseManager,
token,
);
await skeletonQueryWizard.execute();
},
{
title: "Create Query",
},
);
}
/**
* Creates a new `LocalQueryRun` object to track a query evaluation. This creates a timestamp
* file in the query's output directory, creates a `LocalQueryInfo` object, and registers that

View File

@@ -66,8 +66,8 @@ export class QlPackGenerator {
await writeFile(qlPackFilePath, this.header + dump(qlPackYml), "utf8");
}
private async createExampleQlFile() {
const exampleQlFilePath = join(this.folderUri.fsPath, "example.ql");
public async createExampleQlFile(fileName = "example.ql") {
const exampleQlFilePath = join(this.folderUri.fsPath, fileName);
const exampleQl = `
/**

View File

@@ -0,0 +1,295 @@
import { join } from "path";
import { CancellationToken, Uri, workspace, window as Window } from "vscode";
import { CodeQLCliServer } from "./cli";
import { OutputChannelLogger } from "./common";
import { Credentials } from "./common/authentication";
import { QueryLanguage } from "./common/query-language";
import { askForLanguage, isFolderAlreadyInWorkspace } from "./helpers";
import { getErrorMessage } from "./pure/helpers-pure";
import { QlPackGenerator } from "./qlpack-generator";
import { DatabaseItem, DatabaseManager } from "./local-databases";
import * as databaseFetcher from "./databaseFetcher";
import { ProgressCallback, UserCancellationException } from "./progress";
type QueryLanguagesToDatabaseMap = Record<string, string>;
export const QUERY_LANGUAGE_TO_DATABASE_REPO: QueryLanguagesToDatabaseMap = {
cpp: "protocolbuffers/protobuf",
csharp: "dotnet/efcore",
go: "evanw/esbuild",
java: "google/guava",
javascript: "facebook/react",
python: "pallets/flask",
ruby: "rails/rails",
swift: "Alamofire/Alamofire",
};
export class SkeletonQueryWizard {
private language: string | undefined;
private fileName = "example.ql";
private storagePath: string | undefined;
constructor(
private readonly cliServer: CodeQLCliServer,
private readonly progress: ProgressCallback,
private readonly credentials: Credentials | undefined,
private readonly extLogger: OutputChannelLogger,
private readonly databaseManager: DatabaseManager,
private readonly token: CancellationToken,
) {}
private get folderName() {
return `codeql-custom-queries-${this.language}`;
}
public async execute() {
// show quick pick to choose language
this.language = await this.chooseLanguage();
if (!this.language) {
return;
}
this.storagePath = this.getFirstStoragePath();
const skeletonPackAlreadyExists = isFolderAlreadyInWorkspace(
this.folderName,
);
if (skeletonPackAlreadyExists) {
// just create a new example query file in skeleton QL pack
await this.createExampleFile();
// select existing database for language
await this.selectExistingDatabase();
} else {
// generate a new skeleton QL pack with query file
await this.createQlPack();
// download database based on language and select it
await this.downloadDatabase();
}
// open a query file
await this.openExampleFile();
}
private async openExampleFile() {
if (this.folderName === undefined || this.storagePath === undefined) {
throw new Error("Path to folder is undefined");
}
const queryFileUri = Uri.file(
join(this.storagePath, this.folderName, this.fileName),
);
try {
void workspace.openTextDocument(queryFileUri).then((doc) => {
void Window.showTextDocument(doc);
});
} catch (e: unknown) {
void this.extLogger.log(
`Could not open example query file: ${getErrorMessage(e)}`,
);
}
}
public getFirstStoragePath() {
const workspaceFolders = workspace.workspaceFolders;
if (!workspaceFolders || workspaceFolders.length === 0) {
throw new Error("No workspace folders found");
}
const firstFolder = workspaceFolders[0];
// For the vscode-codeql-starter repo, the first folder will be a ql pack
// so we need to get the parent folder
if (firstFolder.uri.path.includes("codeql-custom-queries")) {
// slice off the last part of the path and return the parent folder
return firstFolder.uri.path.split("/").slice(0, -1).join("/");
} else {
// if the first folder is not a ql pack, then we are in a normal workspace
return firstFolder.uri.path;
}
}
private async chooseLanguage() {
this.progress({
message: "Choose language",
step: 1,
maxStep: 1,
});
return await askForLanguage(this.cliServer, false);
}
private async createQlPack() {
if (this.folderName === undefined) {
throw new Error("Folder name is undefined");
}
this.progress({
message: "Creating skeleton QL pack around query",
step: 2,
maxStep: 2,
});
try {
const qlPackGenerator = new QlPackGenerator(
this.folderName,
this.language as QueryLanguage,
this.cliServer,
this.storagePath,
);
await qlPackGenerator.generate();
} catch (e: unknown) {
void this.extLogger.log(
`Could not create skeleton QL pack: ${getErrorMessage(e)}`,
);
}
}
private async createExampleFile() {
if (this.folderName === undefined) {
throw new Error("Folder name is undefined");
}
this.progress({
message:
"Skeleton query pack already exists. Creating additional query example file.",
step: 2,
maxStep: 2,
});
try {
const qlPackGenerator = new QlPackGenerator(
this.folderName,
this.language as QueryLanguage,
this.cliServer,
this.storagePath,
);
this.fileName = await this.determineNextFileName(this.folderName);
await qlPackGenerator.createExampleQlFile(this.fileName);
} catch (e: unknown) {
void this.extLogger.log(
`Could not create skeleton QL pack: ${getErrorMessage(e)}`,
);
}
}
private async determineNextFileName(folderName: string): Promise<string> {
if (this.storagePath === undefined) {
throw new Error("Workspace storage path is undefined");
}
const folderUri = Uri.file(join(this.storagePath, folderName));
const files = await workspace.fs.readDirectory(folderUri);
const qlFiles = files.filter(([filename, _fileType]) =>
filename.match(/example[0-9]*.ql/),
);
return `example${qlFiles.length + 1}.ql`;
}
private async downloadDatabase() {
if (this.storagePath === undefined) {
throw new Error("Workspace storage path is undefined");
}
if (this.language === undefined) {
throw new Error("Language is undefined");
}
this.progress({
message: "Downloading database",
step: 3,
maxStep: 3,
});
const githubRepoNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[this.language];
const chosenRepo = await databaseFetcher.askForGitHubRepo(
undefined,
githubRepoNwo,
);
if (!chosenRepo) {
throw new UserCancellationException("No GitHub repository provided");
}
await databaseFetcher.downloadGitHubDatabase(
chosenRepo,
this.databaseManager,
this.storagePath,
this.credentials,
this.progress,
this.token,
this.cliServer,
this.language,
);
}
private async selectExistingDatabase() {
if (this.language === undefined) {
throw new Error("Language is undefined");
}
if (this.storagePath === undefined) {
throw new Error("Workspace storage path is undefined");
}
const databaseNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[this.language];
const existingDatabaseItem = await this.findDatabaseItemByNwo(
this.language,
databaseNwo,
this.databaseManager.databaseItems,
);
if (existingDatabaseItem) {
// select the found database
await this.databaseManager.setCurrentDatabaseItem(existingDatabaseItem);
} else {
const sameLanguageDatabaseItem = await this.findDatabaseItemByLanguage(
this.language,
this.databaseManager.databaseItems,
);
if (sameLanguageDatabaseItem) {
// select the found database
await this.databaseManager.setCurrentDatabaseItem(
sameLanguageDatabaseItem,
);
} else {
// download new database and select it
await this.downloadDatabase();
}
}
}
public async findDatabaseItemByNwo(
language: string,
databaseNwo: string,
databaseItems: readonly DatabaseItem[],
): Promise<DatabaseItem | undefined> {
const dbItems = databaseItems || [];
const dbs = dbItems.filter(
(db) => db.language === language && db.name === databaseNwo,
);
if (dbs.length === 0) {
return undefined;
}
return dbs[0];
}
public async findDatabaseItemByLanguage(
language: string,
databaseItems: readonly DatabaseItem[],
): Promise<DatabaseItem | undefined> {
const dbItems = databaseItems || [];
const dbs = dbItems.filter((db) => db.language === language);
if (dbs.length === 0) {
return undefined;
}
return dbs[0];
}
}

View File

@@ -0,0 +1,46 @@
import { join } from "path";
import { Uri } from "vscode";
import {
DatabaseContents,
DatabaseItemImpl,
FullDatabaseOptions,
} from "../../../src/local-databases";
import { DirResult } from "tmp";
export function mockDbOptions(): FullDatabaseOptions {
return {
dateAdded: 123,
ignoreSourceArchive: false,
language: "",
};
}
export function createMockDB(
dir: DirResult,
dbOptions = mockDbOptions(),
// source archive location must be a real(-ish) location since
// tests will add this to the workspace location
sourceArchiveUri?: Uri,
databaseUri?: Uri,
): DatabaseItemImpl {
sourceArchiveUri = sourceArchiveUri || sourceLocationUri(dir);
databaseUri = databaseUri || dbLocationUri(dir);
return new DatabaseItemImpl(
databaseUri,
{
sourceArchiveUri,
datasetUri: databaseUri,
} as DatabaseContents,
dbOptions,
() => void 0,
);
}
export function sourceLocationUri(dir: DirResult) {
return Uri.file(join(dir.name, "src.zip"));
}
export function dbLocationUri(dir: DirResult) {
return Uri.file(join(dir.name, "db"));
}

View File

@@ -0,0 +1,418 @@
import { CodeQLCliServer } from "../../../src/cli";
import {
QUERY_LANGUAGE_TO_DATABASE_REPO,
SkeletonQueryWizard,
} from "../../../src/skeleton-query-wizard";
import { mockedObject, mockedQuickPickItem } from "../utils/mocking.helpers";
import * as tmp from "tmp";
import { TextDocument, window, workspace, WorkspaceFolder } from "vscode";
import { extLogger } from "../../../src/common";
import { QlPackGenerator } from "../../../src/qlpack-generator";
import * as helpers from "../../../src/helpers";
import { createFileSync, ensureDirSync, removeSync } from "fs-extra";
import { join } from "path";
import { CancellationTokenSource } from "vscode-jsonrpc";
import { testCredentialsWithStub } from "../../factories/authentication";
import {
DatabaseItem,
DatabaseManager,
FullDatabaseOptions,
} from "../../../src/local-databases";
import * as databaseFetcher from "../../../src/databaseFetcher";
import { createMockDB } from "../../factories/databases/databases";
jest.setTimeout(40_000);
describe("SkeletonQueryWizard", () => {
let mockCli: CodeQLCliServer;
let wizard: SkeletonQueryWizard;
let mockDatabaseManager: DatabaseManager;
let dir: tmp.DirResult;
let storagePath: string;
let quickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
let generateSpy: jest.SpiedFunction<
typeof QlPackGenerator.prototype.generate
>;
let createExampleQlFileSpy: jest.SpiedFunction<
typeof QlPackGenerator.prototype.createExampleQlFile
>;
let downloadGitHubDatabaseSpy: jest.SpiedFunction<
typeof databaseFetcher.downloadGitHubDatabase
>;
let askForGitHubRepoSpy: jest.SpiedFunction<
typeof databaseFetcher.askForGitHubRepo
>;
let openTextDocumentSpy: jest.SpiedFunction<
typeof workspace.openTextDocument
>;
const token = new CancellationTokenSource().token;
const credentials = testCredentialsWithStub();
const chosenLanguage = "ruby";
jest.spyOn(extLogger, "log").mockResolvedValue(undefined);
beforeEach(async () => {
mockCli = mockedObject<CodeQLCliServer>({
resolveLanguages: jest
.fn()
.mockResolvedValue([
"ruby",
"javascript",
"go",
"java",
"python",
"csharp",
"cpp",
]),
getSupportedLanguages: jest.fn(),
});
mockDatabaseManager = mockedObject<DatabaseManager>({
setCurrentDatabaseItem: jest.fn(),
databaseItems: [] as DatabaseItem[],
});
dir = tmp.dirSync({
prefix: "skeleton_query_wizard_",
unsafeCleanup: true,
});
storagePath = dir.name;
jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValue([
{
name: `codespaces-codeql`,
uri: { path: storagePath },
},
{
name: "/second/folder/path",
uri: { path: storagePath },
},
] as WorkspaceFolder[]);
quickPickSpy = jest
.spyOn(window, "showQuickPick")
.mockResolvedValueOnce(mockedQuickPickItem(chosenLanguage));
generateSpy = jest
.spyOn(QlPackGenerator.prototype, "generate")
.mockResolvedValue(undefined);
createExampleQlFileSpy = jest
.spyOn(QlPackGenerator.prototype, "createExampleQlFile")
.mockResolvedValue(undefined);
downloadGitHubDatabaseSpy = jest
.spyOn(databaseFetcher, "downloadGitHubDatabase")
.mockResolvedValue(undefined);
openTextDocumentSpy = jest
.spyOn(workspace, "openTextDocument")
.mockResolvedValue({} as TextDocument);
wizard = new SkeletonQueryWizard(
mockCli,
jest.fn(),
credentials,
extLogger,
mockDatabaseManager,
token,
);
askForGitHubRepoSpy = jest
.spyOn(databaseFetcher, "askForGitHubRepo")
.mockResolvedValue(QUERY_LANGUAGE_TO_DATABASE_REPO[chosenLanguage]);
});
afterEach(async () => {
dir.removeCallback();
});
it("should prompt for language", async () => {
await wizard.execute();
expect(mockCli.getSupportedLanguages).toHaveBeenCalled();
expect(quickPickSpy).toHaveBeenCalled();
});
describe("if QL pack doesn't exist", () => {
beforeEach(() => {
jest.spyOn(helpers, "isFolderAlreadyInWorkspace").mockReturnValue(false);
});
it("should try to create a new QL pack based on the language", async () => {
await wizard.execute();
expect(generateSpy).toHaveBeenCalled();
});
it("should download database for selected language", async () => {
await wizard.execute();
expect(downloadGitHubDatabaseSpy).toHaveBeenCalled();
});
it("should open the query file", async () => {
await wizard.execute();
expect(openTextDocumentSpy).toHaveBeenCalledWith(
expect.objectContaining({
path: expect.stringMatching("example.ql"),
}),
);
});
});
describe("if QL pack exists", () => {
beforeEach(async () => {
jest.spyOn(helpers, "isFolderAlreadyInWorkspace").mockReturnValue(true);
// create a skeleton codeql-custom-queries-${language} folder
// with an example QL file inside
ensureDirSync(
join(dir.name, `codeql-custom-queries-${chosenLanguage}`, "example.ql"),
);
});
it("should create new query file in the same QL pack folder", async () => {
await wizard.execute();
expect(createExampleQlFileSpy).toHaveBeenCalledWith("example2.ql");
});
it("should only take into account example QL files", async () => {
createFileSync(
join(dir.name, `codeql-custom-queries-${chosenLanguage}`, "MyQuery.ql"),
);
await wizard.execute();
expect(createExampleQlFileSpy).toHaveBeenCalledWith("example2.ql");
});
describe("if QL pack has no query file", () => {
it("should create new query file in the same QL pack folder", async () => {
removeSync(
join(
dir.name,
`codeql-custom-queries-${chosenLanguage}`,
"example.ql",
),
);
await wizard.execute();
expect(createExampleQlFileSpy).toHaveBeenCalledWith("example1.ql");
});
it("should open the query file", async () => {
removeSync(
join(
dir.name,
`codeql-custom-queries-${chosenLanguage}`,
"example.ql",
),
);
await wizard.execute();
expect(openTextDocumentSpy).toHaveBeenCalledWith(
expect.objectContaining({
path: expect.stringMatching("example1.ql"),
}),
);
});
});
describe("if database is also already downloaded", () => {
let databaseNwo: string;
let databaseItem: DatabaseItem;
let mockDatabaseManagerWithItems: DatabaseManager;
beforeEach(async () => {
databaseNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[chosenLanguage];
databaseItem = {
name: databaseNwo,
language: chosenLanguage,
} as DatabaseItem;
mockDatabaseManagerWithItems = mockedObject<DatabaseManager>({
setCurrentDatabaseItem: jest.fn(),
databaseItems: [databaseItem] as DatabaseItem[],
});
wizard = new SkeletonQueryWizard(
mockCli,
jest.fn(),
credentials,
extLogger,
mockDatabaseManagerWithItems,
token,
);
});
it("should not download a new database for language", async () => {
await wizard.execute();
expect(downloadGitHubDatabaseSpy).not.toHaveBeenCalled();
});
it("should select an existing database", async () => {
await wizard.execute();
expect(
mockDatabaseManagerWithItems.setCurrentDatabaseItem,
).toHaveBeenCalledWith(databaseItem);
});
it("should open the new query file", async () => {
await wizard.execute();
expect(openTextDocumentSpy).toHaveBeenCalledWith(
expect.objectContaining({
path: expect.stringMatching("example2.ql"),
}),
);
});
});
describe("if database is missing", () => {
describe("if the user choses to downloaded the suggested database from GitHub", () => {
it("should download a new database for language", async () => {
await wizard.execute();
expect(askForGitHubRepoSpy).toHaveBeenCalled();
expect(downloadGitHubDatabaseSpy).toHaveBeenCalled();
});
});
describe("if the user choses to download a different database from GitHub than the one suggested", () => {
beforeEach(() => {
const chosenGitHubRepo = "pickles-owner/pickles-repo";
askForGitHubRepoSpy = jest
.spyOn(databaseFetcher, "askForGitHubRepo")
.mockResolvedValue(chosenGitHubRepo);
});
it("should download the newly chosen database", async () => {
await wizard.execute();
expect(askForGitHubRepoSpy).toHaveBeenCalled();
expect(downloadGitHubDatabaseSpy).toHaveBeenCalled();
});
});
});
});
describe("getFirstStoragePath", () => {
it("should return the first workspace folder", async () => {
jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValue([
{
name: "codeql-custom-queries-cpp",
uri: { path: "codespaces-codeql" },
},
] as WorkspaceFolder[]);
wizard = new SkeletonQueryWizard(
mockCli,
jest.fn(),
credentials,
extLogger,
mockDatabaseManager,
token,
);
expect(wizard.getFirstStoragePath()).toEqual("codespaces-codeql");
});
describe("if user is in vscode-codeql-starter workspace", () => {
it("should set storage path to parent folder", async () => {
jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValue([
{
name: "codeql-custom-queries-cpp",
uri: { path: "vscode-codeql-starter/codeql-custom-queries-cpp" },
},
{
name: "codeql-custom-queries-csharp",
uri: { path: "vscode-codeql-starter/codeql-custom-queries-csharp" },
},
] as WorkspaceFolder[]);
wizard = new SkeletonQueryWizard(
mockCli,
jest.fn(),
credentials,
extLogger,
mockDatabaseManager,
token,
);
expect(wizard.getFirstStoragePath()).toEqual("vscode-codeql-starter");
});
});
});
describe("findDatabaseItemByNwo", () => {
describe("when the item exists", () => {
it("should return the database item", async () => {
const mockDbItem = createMockDB(dir);
const mockDbItem2 = createMockDB(dir);
const databaseItem = await wizard.findDatabaseItemByNwo(
mockDbItem.language,
mockDbItem.name,
[mockDbItem, mockDbItem2],
);
expect(databaseItem!.language).toEqual(mockDbItem.language);
expect(databaseItem!.name).toEqual(mockDbItem.name);
});
});
describe("when the item doesn't exist", () => {
it("should return nothing", async () => {
const mockDbItem = createMockDB(dir);
const mockDbItem2 = createMockDB(dir);
const databaseItem = await wizard.findDatabaseItemByNwo(
"ruby",
"mock-nwo",
[mockDbItem, mockDbItem2],
);
expect(databaseItem).toBeUndefined();
});
});
});
describe("findDatabaseItemByLanguage", () => {
describe("when the item exists", () => {
it("should return the database item", async () => {
const mockDbItem = createMockDB(dir, {
language: "ruby",
} as FullDatabaseOptions);
const mockDbItem2 = createMockDB(dir, {
language: "javascript",
} as FullDatabaseOptions);
const databaseItem = await wizard.findDatabaseItemByLanguage("ruby", [
mockDbItem,
mockDbItem2,
]);
expect(databaseItem).toEqual(mockDbItem);
});
});
describe("when the item doesn't exist", () => {
it("should return nothing", async () => {
const mockDbItem = createMockDB(dir);
const mockDbItem2 = createMockDB(dir);
const databaseItem = await wizard.findDatabaseItemByLanguage("ruby", [
mockDbItem,
mockDbItem2,
]);
expect(databaseItem).toBeUndefined();
});
});
});
});

View File

@@ -4,7 +4,6 @@ import { join } from "path";
import { CancellationToken, ExtensionContext, Uri, workspace } from "vscode";
import {
DatabaseContents,
DatabaseContentsWithDbScheme,
DatabaseEventKind,
DatabaseItemImpl,
@@ -27,14 +26,14 @@ import { Setting } from "../../../src/config";
import { QlPackGenerator } from "../../../src/qlpack-generator";
import { mockedObject } from "../utils/mocking.helpers";
import { createMockApp } from "../../__mocks__/appMock";
import {
createMockDB,
dbLocationUri,
mockDbOptions,
sourceLocationUri,
} from "../../factories/databases/databases";
describe("local databases", () => {
const MOCK_DB_OPTIONS: FullDatabaseOptions = {
dateAdded: 123,
ignoreSourceArchive: false,
language: "",
};
let databaseManager: DatabaseManager;
let extensionContext: ExtensionContext;
@@ -118,7 +117,7 @@ describe("local databases", () => {
});
it("should fire events when adding and removing a db item", async () => {
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
const onDidChangeDatabaseItem = jest.fn();
databaseManager.onDidChangeDatabaseItem(onDidChangeDatabaseItem);
await (databaseManager as any).addDatabaseItem(
@@ -130,8 +129,8 @@ describe("local databases", () => {
expect((databaseManager as any)._databaseItems).toEqual([mockDbItem]);
expect(updateSpy).toBeCalledWith("databaseList", [
{
options: MOCK_DB_OPTIONS,
uri: dbLocationUri().toString(true),
options: mockDbOptions(),
uri: dbLocationUri(dir).toString(true),
},
]);
expect(onDidChangeDatabaseItem).toBeCalledWith({
@@ -158,7 +157,7 @@ describe("local databases", () => {
describe("renameDatabaseItem", () => {
it("should rename a db item and emit an event", async () => {
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
const onDidChangeDatabaseItem = jest.fn();
databaseManager.onDidChangeDatabaseItem(onDidChangeDatabaseItem);
await (databaseManager as any).addDatabaseItem(
@@ -172,8 +171,8 @@ describe("local databases", () => {
expect(mockDbItem.name).toBe("new name");
expect(updateSpy).toBeCalledWith("databaseList", [
{
options: { ...MOCK_DB_OPTIONS, displayName: "new name" },
uri: dbLocationUri().toString(true),
options: { ...mockDbOptions(), displayName: "new name" },
uri: dbLocationUri(dir).toString(true),
},
]);
@@ -188,7 +187,7 @@ describe("local databases", () => {
it("should add a database item", async () => {
const onDidChangeDatabaseItem = jest.fn();
databaseManager.onDidChangeDatabaseItem(onDidChangeDatabaseItem);
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
await (databaseManager as any).addDatabaseItem(
{} as ProgressCallback,
@@ -199,8 +198,8 @@ describe("local databases", () => {
expect(databaseManager.databaseItems).toEqual([mockDbItem]);
expect(updateSpy).toBeCalledWith("databaseList", [
{
uri: dbLocationUri().toString(true),
options: MOCK_DB_OPTIONS,
uri: dbLocationUri(dir).toString(true),
options: mockDbOptions(),
},
]);
@@ -212,7 +211,7 @@ describe("local databases", () => {
});
it("should add a database item source archive", async () => {
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
mockDbItem.name = "xxx";
await databaseManager.addDatabaseSourceArchiveFolder(mockDbItem);
@@ -223,13 +222,13 @@ describe("local databases", () => {
// must use a matcher here since vscode URIs with the same path
// are not always equal due to internal state.
uri: expect.objectContaining({
fsPath: encodeArchiveBasePath(sourceLocationUri().fsPath).fsPath,
fsPath: encodeArchiveBasePath(sourceLocationUri(dir).fsPath).fsPath,
}),
});
});
it("should remove a database item", async () => {
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
await fs.ensureDir(mockDbItem.databaseUri.fsPath);
// pretend that this item is the first workspace folder in the list
@@ -263,7 +262,7 @@ describe("local databases", () => {
});
it("should remove a database item outside of the extension controlled area", async () => {
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
await fs.ensureDir(mockDbItem.databaseUri.fsPath);
// pretend that this item is the first workspace folder in the list
@@ -301,7 +300,7 @@ describe("local databases", () => {
it("should register and deregister a database when adding and removing it", async () => {
// similar test as above, but also check the call to sendRequestSpy to make sure they send the
// registration messages.
const mockDbItem = createMockDB();
const mockDbItem = createMockDB(dir);
await (databaseManager as any).addDatabaseItem(
{} as ProgressCallback,
@@ -325,7 +324,8 @@ describe("local databases", () => {
describe("resolveSourceFile", () => {
it("should fail to resolve when not a uri", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
@@ -336,7 +336,8 @@ describe("local databases", () => {
it("should fail to resolve when not a file uri", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
@@ -348,17 +349,19 @@ describe("local databases", () => {
describe("no source archive", () => {
it("should resolve undefined", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
const resolved = db.resolveSourceFile(undefined);
expect(resolved.toString(true)).toBe(dbLocationUri().toString(true));
expect(resolved.toString(true)).toBe(dbLocationUri(dir).toString(true));
});
it("should resolve an empty file", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
@@ -370,7 +373,8 @@ describe("local databases", () => {
describe("zipped source archive", () => {
it("should encode a source archive url", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
encodeSourceArchiveUri({
sourceArchiveZipPath: "sourceArchive-uri",
pathWithinSourceArchive: "def",
@@ -390,7 +394,8 @@ describe("local databases", () => {
it("should encode a source archive url with trailing slash", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
encodeSourceArchiveUri({
sourceArchiveZipPath: "sourceArchive-uri",
pathWithinSourceArchive: "def/",
@@ -410,7 +415,8 @@ describe("local databases", () => {
it("should encode an empty source archive url", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
encodeSourceArchiveUri({
sourceArchiveZipPath: "sourceArchive-uri",
pathWithinSourceArchive: "def",
@@ -425,7 +431,8 @@ describe("local databases", () => {
it("should handle an empty file", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
dir,
mockDbOptions(),
Uri.parse("file:/sourceArchive-uri/"),
);
const resolved = db.resolveSourceFile("");
@@ -471,8 +478,9 @@ describe("local databases", () => {
it("should return true for testproj database in test directory", async () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(directoryPath)).toBe(true);
@@ -480,8 +488,9 @@ describe("local databases", () => {
it("should return false for non-existent test directory", async () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(join(dir.name, "non-existent/non-existent.testproj")),
);
expect(await db.isAffectedByTest(join(dir.name, "non-existent"))).toBe(
@@ -494,8 +503,9 @@ describe("local databases", () => {
await fs.writeFile(anotherProjectPath, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(anotherProjectPath),
);
expect(await db.isAffectedByTest(directoryPath)).toBe(false);
@@ -508,8 +518,9 @@ describe("local databases", () => {
await fs.writeFile(anotherProjectPath, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(anotherProjectPath),
);
expect(await db.isAffectedByTest(directoryPath)).toBe(false);
@@ -517,8 +528,9 @@ describe("local databases", () => {
it("should return false for testproj database for prefix directory", async () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(projectPath),
);
// /d is a prefix of /dir/dir.testproj, but
@@ -528,8 +540,9 @@ describe("local databases", () => {
it("should return true for testproj database for test file", async () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(qlFilePath)).toBe(true);
@@ -538,8 +551,9 @@ describe("local databases", () => {
it("should return false for non-existent test file", async () => {
const otherTestFile = join(directoryPath, "other-test.ql");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(otherTestFile)).toBe(false);
@@ -550,8 +564,9 @@ describe("local databases", () => {
await fs.writeFile(anotherProjectPath, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(anotherProjectPath),
);
expect(await db.isAffectedByTest(qlFilePath)).toBe(false);
@@ -562,8 +577,9 @@ describe("local databases", () => {
await fs.writeFile(otherTestFile, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
dir,
mockDbOptions(),
sourceLocationUri(dir),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(otherTestFile)).toBe(false);
@@ -622,7 +638,7 @@ describe("local databases", () => {
ignoreSourceArchive: false,
language,
};
mockDbItem = createMockDB(options);
mockDbItem = createMockDB(dir, options);
generateSpy = jest
.spyOn(QlPackGenerator.prototype, "generate")
@@ -655,7 +671,7 @@ describe("local databases", () => {
describe("when the language is not set", () => {
it("should fail gracefully", async () => {
mockDbItem = createMockDB();
mockDbItem = createMockDB(dir);
await (databaseManager as any).createSkeletonPacks(mockDbItem);
expect(logSpy).toHaveBeenCalledWith(
"Could not create skeleton QL pack because the selected database's language is not set.",
@@ -701,7 +717,7 @@ describe("local databases", () => {
},
}));
mockDbItem = createMockDB();
mockDbItem = createMockDB(dir);
});
it("should resolve the database contents", async () => {
@@ -784,30 +800,4 @@ describe("local databases", () => {
});
});
});
function createMockDB(
mockDbOptions = MOCK_DB_OPTIONS,
// source archive location must be a real(-ish) location since
// tests will add this to the workspace location
sourceArchiveUri = sourceLocationUri(),
databaseUri = dbLocationUri(),
): DatabaseItemImpl {
return new DatabaseItemImpl(
databaseUri,
{
sourceArchiveUri,
datasetUri: databaseUri,
} as DatabaseContents,
mockDbOptions,
() => void 0,
);
}
function sourceLocationUri() {
return Uri.file(join(dir.name, "src.zip"));
}
function dbLocationUri() {
return Uri.file(join(dir.name, "db"));
}
});

View File

@@ -23,6 +23,48 @@ describe("databaseFetcher", () => {
request: mockRequest,
} as unknown as Octokit.Octokit;
// We can't make the real octokit request (since we need credentials), so we mock the response.
const successfullMockApiResponse = {
data: [
{
id: 1495869,
name: "csharp-database",
language: "csharp",
uploader: {},
content_type: "application/zip",
state: "uploaded",
size: 55599715,
created_at: "2022-03-24T10:46:24Z",
updated_at: "2022-03-24T10:46:27Z",
url: "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/csharp",
},
{
id: 1100671,
name: "database.zip",
language: "javascript",
uploader: {},
content_type: "application/zip",
state: "uploaded",
size: 29294434,
created_at: "2022-03-01T16:00:04Z",
updated_at: "2022-03-01T16:00:06Z",
url: "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/javascript",
},
{
id: 648738,
name: "ql-database",
language: "ql",
uploader: {},
content_type: "application/json; charset=utf-8",
state: "uploaded",
size: 39735500,
created_at: "2022-02-02T09:38:50Z",
updated_at: "2022-02-02T09:38:51Z",
url: "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/ql",
},
],
};
beforeEach(() => {
quickPickSpy = jest
.spyOn(window, "showQuickPick")
@@ -30,48 +72,7 @@ describe("databaseFetcher", () => {
});
it("should convert a GitHub nwo to a database url", async () => {
// We can't make the real octokit request (since we need credentials), so we mock the response.
const mockApiResponse = {
data: [
{
id: 1495869,
name: "csharp-database",
language: "csharp",
uploader: {},
content_type: "application/zip",
state: "uploaded",
size: 55599715,
created_at: "2022-03-24T10:46:24Z",
updated_at: "2022-03-24T10:46:27Z",
url: "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/csharp",
},
{
id: 1100671,
name: "database.zip",
language: "javascript",
uploader: {},
content_type: "application/zip",
state: "uploaded",
size: 29294434,
created_at: "2022-03-01T16:00:04Z",
updated_at: "2022-03-01T16:00:06Z",
url: "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/javascript",
},
{
id: 648738,
name: "ql-database",
language: "ql",
uploader: {},
content_type: "application/json; charset=utf-8",
state: "uploaded",
size: 39735500,
created_at: "2022-02-02T09:38:50Z",
updated_at: "2022-02-02T09:38:51Z",
url: "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/ql",
},
],
};
mockRequest.mockResolvedValue(mockApiResponse);
mockRequest.mockResolvedValue(successfullMockApiResponse);
quickPickSpy.mockResolvedValue(mockedQuickPickItem("javascript"));
const githubRepo = "github/codeql";
const result = await convertGithubNwoToDatabaseUrl(
@@ -127,6 +128,45 @@ describe("databaseFetcher", () => {
).rejects.toThrow(/Unable to get database/);
expect(progressSpy).toBeCalledTimes(1);
});
describe("when language is already provided", () => {
describe("when language is valid", () => {
it("should not prompt the user", async () => {
mockRequest.mockResolvedValue(successfullMockApiResponse);
const githubRepo = "github/codeql";
await convertGithubNwoToDatabaseUrl(
githubRepo,
octokit,
progressSpy,
"javascript",
);
expect(quickPickSpy).not.toHaveBeenCalled();
});
});
describe("when language is invalid", () => {
it("should prompt for language", async () => {
mockRequest.mockResolvedValue(successfullMockApiResponse);
const githubRepo = "github/codeql";
await convertGithubNwoToDatabaseUrl(
githubRepo,
octokit,
progressSpy,
"invalid-language",
);
expect(quickPickSpy).toHaveBeenCalled();
});
});
});
describe("when language is not provided", () => {
it("should prompt for language", async () => {
mockRequest.mockResolvedValue(successfullMockApiResponse);
const githubRepo = "github/codeql";
await convertGithubNwoToDatabaseUrl(githubRepo, octokit, progressSpy);
expect(quickPickSpy).toHaveBeenCalled();
});
});
});
describe("findDirWithFile", () => {