diff --git a/.github/codeql/queries/assert-pure.ql b/.github/codeql/queries/assert-pure.ql index cd840d502..a4805b3e5 100644 --- a/.github/codeql/queries/assert-pure.ql +++ b/.github/codeql/queries/assert-pure.ql @@ -1,21 +1,40 @@ /** * @name Unwanted dependency on vscode API - * @kind problem + * @kind path-problem * @problem.severity error * @id vscode-codeql/assert-pure * @description The modules stored under `pure` and tested in the `pure-tests` * are intended to be "pure". */ + import javascript -class VSCodeImport extends ASTNode { - VSCodeImport() { - this.(Import).getImportedPath().getValue() = "vscode" +class VSCodeImport extends ImportDeclaration { + VSCodeImport() { this.getImportedPath().getValue() = "vscode" } +} + +class PureFile extends File { + PureFile() { + ( + this.getRelativePath().regexpMatch(".*/src/pure/.*") or + this.getRelativePath().regexpMatch(".*/src/common/.*") + ) and + not this.getRelativePath().regexpMatch(".*/vscode/.*") } } +Import getANonTypeOnlyImport(Module m) { + result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly() +} + +query predicate edges(AstNode a, AstNode b) { + getANonTypeOnlyImport(a) = b or + a.(Import).getImportedModule() = b +} + from Module m, VSCodeImport v where - m.getFile().getRelativePath().regexpMatch(".*src/pure/.*") and - m.getAnImportedModule*().getAnImport() = v -select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here" + m.getFile() instanceof PureFile and + edges+(m, v) +select m, m, v, + "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here" diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index 767313d68..2202d0771 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -3,6 +3,10 @@ ## [UNRELEASED] - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392) +- Fix bug to ensure error messages have complete stack trace in message logs. [#2425](https://github.com/github/vscode-codeql/pull/2425) +- Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422) +- Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427) +- Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428) ## 1.8.4 - 3 May 2023 diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 87788fe8d..00618f1aa 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -516,6 +516,10 @@ "title": "Add new list", "icon": "$(new-folder)" }, + { + "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch", + "title": "Add repositories with GitHub Code Search" + }, { "command": "codeQLVariantAnalysisRepositories.setSelectedItem", "title": "Select" @@ -961,6 +965,11 @@ "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/", "group": "2_qlContextMenu@1" }, + { + "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch", + "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/", + "group": "2_qlContextMenu@1" + }, { "command": "codeQLDatabases.setCurrentDatabase", "group": "inline", @@ -1297,6 +1306,10 @@ "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu", "when": "false" }, + { + "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch", + "when": "false" + }, { "command": "codeQLDatabases.setCurrentDatabase", "when": "false" @@ -1593,6 +1606,10 @@ "view": "codeQLQueryHistory", "contents": "You have no query history items at the moment.\n\nSelect a database to run a CodeQL query and get your first results." }, + { + "view": "codeQLQueries", + "contents": "This workspace doesn't contain any CodeQL queries at the moment." + }, { "view": "codeQLDatabases", "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)" diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts index d5ef116d4..16578430c 100644 --- a/extensions/ql-vscode/src/codeql-cli/cli.ts +++ b/extensions/ql-vscode/src/codeql-cli/cli.ts @@ -134,6 +134,11 @@ export interface SourceInfo { sourceLocationPrefix: string; } +/** + * The expected output of `codeql resolve queries`. + */ +export type ResolvedQueries = string[]; + /** * The expected output of `codeql resolve tests`. */ @@ -213,7 +218,7 @@ export class CodeQLCliServer implements Disposable { private readonly app: App, private distributionProvider: DistributionProvider, private cliConfig: CliConfig, - private logger: Logger, + public readonly logger: Logger, ) { this.commandQueue = []; this.commandInProcess = false; @@ -325,6 +330,7 @@ export class CodeQLCliServer implements Disposable { commandArgs: string[], description: string, onLine?: OnLineCallback, + silent?: boolean, ): Promise { const stderrBuffers: Buffer[] = []; if (this.commandInProcess) { @@ -344,7 +350,12 @@ export class CodeQLCliServer implements Disposable { // Compute the full args array const args = command.concat(LOGGING_FLAGS).concat(commandArgs); const argsString = args.join(" "); - void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`); + // If we are running silently, we don't want to print anything to the console. + if (!silent) { + void this.logger.log( + `${description} using CodeQL CLI: ${argsString}...`, + ); + } try { await new Promise((resolve, reject) => { // Start listening to stdout @@ -390,24 +401,30 @@ export class CodeQLCliServer implements Disposable { const fullBuffer = Buffer.concat(stdoutBuffers); // Make sure we remove the terminator; const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1); - void this.logger.log("CLI command succeeded."); + if (!silent) { + void this.logger.log("CLI command succeeded."); + } return data; } catch (err) { // Kill the process if it isn't already dead. this.killProcessIfRunning(); - // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error) + // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error) const newError = stderrBuffers.length === 0 - ? new Error(`${description} failed: ${err}`) + ? new Error( + `${description} failed with args:${EOL} ${argsString}${EOL}${err}`, + ) : new Error( - `${description} failed: ${Buffer.concat(stderrBuffers).toString( - "utf8", - )}`, + `${description} failed with args:${EOL} ${argsString}${EOL}${Buffer.concat( + stderrBuffers, + ).toString("utf8")}`, ); newError.stack += getErrorStack(err); throw newError; } finally { - void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8")); + if (!silent) { + void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8")); + } // Remove the listeners we set up. process.stdout.removeAllListeners("data"); process.stderr.removeAllListeners("data"); @@ -544,9 +561,11 @@ export class CodeQLCliServer implements Disposable { { progressReporter, onLine, + silent = false, }: { progressReporter?: ProgressReporter; onLine?: OnLineCallback; + silent?: boolean; } = {}, ): Promise { if (progressReporter) { @@ -562,6 +581,7 @@ export class CodeQLCliServer implements Disposable { commandArgs, description, onLine, + silent, ).then(resolve, reject); } catch (err) { reject(err); @@ -595,10 +615,12 @@ export class CodeQLCliServer implements Disposable { addFormat = true, progressReporter, onLine, + silent = false, }: { addFormat?: boolean; progressReporter?: ProgressReporter; onLine?: OnLineCallback; + silent?: boolean; } = {}, ): Promise { let args: string[] = []; @@ -609,6 +631,7 @@ export class CodeQLCliServer implements Disposable { const result = await this.runCodeQlCliCommand(command, args, description, { progressReporter, onLine, + silent, }); try { return JSON.parse(result) as OutputType; @@ -731,6 +754,25 @@ export class CodeQLCliServer implements Disposable { ); } + /** + * Finds all available queries in a given directory. + * @param queryDir Root of directory tree to search for queries. + * @param silent If true, don't print logs to the CodeQL extension log. + * @returns The list of queries that were found. + */ + public async resolveQueries( + queryDir: string, + silent?: boolean, + ): Promise { + const subcommandArgs = [queryDir]; + return await this.runJsonCodeQlCliCommand( + ["resolve", "queries"], + subcommandArgs, + "Resolving queries", + { silent }, + ); + } + /** * Finds all available QL tests in a given directory. * @param testPath Root of directory tree to search for tests. @@ -1031,6 +1073,7 @@ export class CodeQLCliServer implements Disposable { resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo, + args?: string[], ): Promise { const additionalArgs = [ // TODO: This flag means that we don't group interpreted results @@ -1038,6 +1081,7 @@ export class CodeQLCliServer implements Disposable { // interpretation with and without this flag, or do some // grouping client-side. "--no-group-results", + ...(args ?? []), ]; await this.runInterpretCommand( @@ -1737,6 +1781,10 @@ export function shouldDebugCliServer() { } export class CliVersionConstraint { + // The oldest version of the CLI that we support. This is used to determine + // whether to show a warning about the CLI being too old on startup. + public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.7.6"); + /** * CLI version where building QLX packs for remote queries is supported. * (The options were _accepted_ by a few earlier versions, but only from @@ -1795,6 +1843,8 @@ export class CliVersionConstraint { "2.12.4", ); + public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4"); + constructor(private readonly cli: CodeQLCliServer) { /**/ } @@ -1864,4 +1914,8 @@ export class CliVersionConstraint { CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL, ); } + + async usesGlobalCompilationCache() { + return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE); + } } diff --git a/extensions/ql-vscode/src/codeql-cli/distribution.ts b/extensions/ql-vscode/src/codeql-cli/distribution.ts index b64e73272..7c98b3aaa 100644 --- a/extensions/ql-vscode/src/codeql-cli/distribution.ts +++ b/extensions/ql-vscode/src/codeql-cli/distribution.ts @@ -6,12 +6,7 @@ import * as semver from "semver"; import { URL } from "url"; import { ExtensionContext, Event } from "vscode"; import { DistributionConfig } from "../config"; -import { - InvocationRateLimiter, - InvocationRateLimiterResultKind, - showAndLogErrorMessage, - showAndLogWarningMessage, -} from "../helpers"; +import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers"; import { extLogger } from "../common"; import { getCodeQlCliVersion } from "./cli-version"; import { @@ -24,6 +19,10 @@ import { extractZipArchive, getRequiredAssetName, } from "../pure/distribution"; +import { + InvocationRateLimiter, + InvocationRateLimiterResultKind, +} from "../common/invocation-rate-limiter"; /** * distribution.ts @@ -76,7 +75,7 @@ export class DistributionManager implements DistributionProvider { extensionContext, ); this.updateCheckRateLimiter = new InvocationRateLimiter( - extensionContext, + extensionContext.globalState, "extensionSpecificDistributionUpdateCheck", () => this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(), diff --git a/extensions/ql-vscode/src/common/app.ts b/extensions/ql-vscode/src/common/app.ts index 2e749c018..f50eb729d 100644 --- a/extensions/ql-vscode/src/common/app.ts +++ b/extensions/ql-vscode/src/common/app.ts @@ -4,6 +4,11 @@ import { AppEventEmitter } from "./events"; import { Logger } from "./logging"; import { Memento } from "./memento"; import { AppCommandManager } from "./commands"; +import type { + WorkspaceFolder, + Event, + WorkspaceFoldersChangeEvent, +} from "vscode"; export interface App { createEventEmitter(): AppEventEmitter; @@ -14,6 +19,8 @@ export interface App { readonly globalStoragePath: string; readonly workspaceStoragePath?: string; readonly workspaceState: Memento; + readonly workspaceFolders: readonly WorkspaceFolder[] | undefined; + readonly onDidChangeWorkspaceFolders: Event; readonly credentials: Credentials; readonly commands: AppCommandManager; } diff --git a/extensions/ql-vscode/src/common/commands.ts b/extensions/ql-vscode/src/common/commands.ts index bbd8c7ece..74d5560af 100644 --- a/extensions/ql-vscode/src/common/commands.ts +++ b/extensions/ql-vscode/src/common/commands.ts @@ -251,6 +251,9 @@ export type VariantAnalysisCommands = { "codeQL.monitorRehydratedVariantAnalysis": ( variantAnalysis: VariantAnalysis, ) => Promise; + "codeQL.monitorReauthenticatedVariantAnalysis": ( + variantAnalysis: VariantAnalysis, + ) => Promise; "codeQL.openVariantAnalysisLogs": ( variantAnalysisId: number, ) => Promise; @@ -272,6 +275,7 @@ export type DatabasePanelCommands = { "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction; "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction; "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction; + "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction; }; export type AstCfgCommands = { diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts index 8c9f970d5..6f2656a2f 100644 --- a/extensions/ql-vscode/src/common/discovery.ts +++ b/extensions/ql-vscode/src/common/discovery.ts @@ -1,6 +1,6 @@ import { DisposableObject } from "../pure/disposable-object"; -import { extLogger } from "./logging/vscode/loggers"; import { getErrorMessage } from "../pure/helpers-pure"; +import { Logger } from "./logging"; /** * Base class for "discovery" operations, which scan the file system to find specific kinds of @@ -8,18 +8,28 @@ import { getErrorMessage } from "../pure/helpers-pure"; * same time. */ export abstract class Discovery extends DisposableObject { - private retry = false; - private discoveryInProgress = false; + private restartWhenFinished = false; + private currentDiscoveryPromise: Promise | undefined; - constructor(private readonly name: string) { + constructor(private readonly name: string, private readonly logger: Logger) { super(); } + /** + * Returns the promise of the currently running refresh operation, if one is in progress. + * Otherwise returns a promise that resolves immediately. + */ + public waitForCurrentRefresh(): Promise { + return this.currentDiscoveryPromise ?? Promise.resolve(); + } + /** * Force the discovery process to run. Normally invoked by the derived class when a relevant file * system change is detected. + * + * Returns a promise that resolves when the refresh is complete, including any retries. */ - public refresh(): void { + public refresh(): Promise { // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we // got a storm of refresh requests due to, say, the copying or deletion of a large directory // tree, we could potentially spawn a separate simultaneous discovery operation for each @@ -36,14 +46,16 @@ export abstract class Discovery extends DisposableObject { // other change notifications that might be coming along. However, this would create more // latency in the common case, in order to save a bit of latency in the uncommon case. - if (this.discoveryInProgress) { + if (this.currentDiscoveryPromise !== undefined) { // There's already a discovery operation in progress. Tell it to restart when it's done. - this.retry = true; + this.restartWhenFinished = true; } else { // No discovery in progress, so start one now. - this.discoveryInProgress = true; - this.launchDiscovery(); + this.currentDiscoveryPromise = this.launchDiscovery().finally(() => { + this.currentDiscoveryPromise = undefined; + }); } + return this.currentDiscoveryPromise; } /** @@ -51,34 +63,31 @@ export abstract class Discovery extends DisposableObject { * discovery operation completes, the `update` function will be invoked with the results of the * discovery. */ - private launchDiscovery(): void { - const discoveryPromise = this.discover(); - discoveryPromise - .then((results) => { - if (!this.retry) { - // Update any listeners with the results of the discovery. - this.discoveryInProgress = false; - this.update(results); - } - }) + private async launchDiscovery(): Promise { + let results: T | undefined; + try { + results = await this.discover(); + } catch (err) { + void this.logger.log( + `${this.name} failed. Reason: ${getErrorMessage(err)}`, + ); + results = undefined; + } - .catch((err: unknown) => { - void extLogger.log( - `${this.name} failed. Reason: ${getErrorMessage(err)}`, - ); - }) - - .finally(() => { - if (this.retry) { - // Another refresh request came in while we were still running a previous discovery - // operation. Since the discovery results we just computed are now stale, we'll launch - // another discovery operation instead of updating. - // Note that by doing this inside of `finally`, we will relaunch discovery even if the - // initial discovery operation failed. - this.retry = false; - this.launchDiscovery(); - } - }); + if (this.restartWhenFinished) { + // Another refresh request came in while we were still running a previous discovery + // operation. Since the discovery results we just computed are now stale, we'll launch + // another discovery operation instead of updating. + // We want to relaunch discovery regardless of if the initial discovery operation + // succeeded or failed. + this.restartWhenFinished = false; + await this.launchDiscovery(); + } else { + // If the discovery was successful, then update any listeners with the results. + if (results !== undefined) { + this.update(results); + } + } } /** diff --git a/extensions/ql-vscode/src/common/events.ts b/extensions/ql-vscode/src/common/events.ts index 67e457405..16ba4821c 100644 --- a/extensions/ql-vscode/src/common/events.ts +++ b/extensions/ql-vscode/src/common/events.ts @@ -4,7 +4,7 @@ export interface AppEvent { (listener: (event: T) => void): Disposable; } -export interface AppEventEmitter { +export interface AppEventEmitter extends Disposable { event: AppEvent; fire(data: T): void; } diff --git a/extensions/ql-vscode/src/common/file-tree-nodes.ts b/extensions/ql-vscode/src/common/file-tree-nodes.ts new file mode 100644 index 000000000..3544265ca --- /dev/null +++ b/extensions/ql-vscode/src/common/file-tree-nodes.ts @@ -0,0 +1,106 @@ +import { basename, dirname, join } from "path"; +import { env } from "vscode"; + +/** + * A node in the tree of files. This will be either a `FileTreeDirectory` or a `FileTreeLeaf`. + */ +export abstract class FileTreeNode { + constructor(private _path: string, private _name: string) {} + + public get path(): string { + return this._path; + } + + public get name(): string { + return this._name; + } + + public abstract get children(): readonly FileTreeNode[]; + + public abstract finish(): void; +} + +/** + * A directory containing one or more files or other directories. + */ +export class FileTreeDirectory extends FileTreeNode { + constructor( + _path: string, + _name: string, + private _children: FileTreeNode[] = [], + ) { + super(_path, _name); + } + + public get children(): readonly FileTreeNode[] { + return this._children; + } + + public addChild(child: FileTreeNode): void { + this._children.push(child); + } + + public createDirectory(relativePath: string): FileTreeDirectory { + if (relativePath === ".") { + return this; + } + const dirName = dirname(relativePath); + if (dirName === ".") { + return this.createChildDirectory(relativePath); + } else { + const parent = this.createDirectory(dirName); + return parent.createDirectory(basename(relativePath)); + } + } + + public finish(): void { + // remove empty directories + this._children.filter( + (child) => child instanceof FileTreeLeaf || child.children.length > 0, + ); + this._children.sort((a, b) => a.name.localeCompare(b.name, env.language)); + this._children.forEach((child, i) => { + child.finish(); + if ( + child.children?.length === 1 && + child.children[0] instanceof FileTreeDirectory + ) { + // collapse children + const replacement = new FileTreeDirectory( + child.children[0].path, + `${child.name} / ${child.children[0].name}`, + Array.from(child.children[0].children), + ); + this._children[i] = replacement; + } + }); + } + + private createChildDirectory(name: string): FileTreeDirectory { + const existingChild = this._children.find((child) => child.name === name); + if (existingChild !== undefined) { + return existingChild as FileTreeDirectory; + } else { + const newChild = new FileTreeDirectory(join(this.path, name), name); + this.addChild(newChild); + return newChild; + } + } +} + +/** + * A single file. + */ +export class FileTreeLeaf extends FileTreeNode { + constructor(_path: string, _name: string) { + super(_path, _name); + } + + public get children(): readonly FileTreeNode[] { + return []; + } + + public finish(): void { + /**/ + } +} diff --git a/extensions/ql-vscode/src/common/invocation-rate-limiter.ts b/extensions/ql-vscode/src/common/invocation-rate-limiter.ts new file mode 100644 index 000000000..325a5df18 --- /dev/null +++ b/extensions/ql-vscode/src/common/invocation-rate-limiter.ts @@ -0,0 +1,89 @@ +import { Memento } from "./memento"; + +/** + * Provides a utility method to invoke a function only if a minimum time interval has elapsed since + * the last invocation of that function. + */ +export class InvocationRateLimiter { + constructor( + private readonly globalState: Memento, + private readonly funcIdentifier: string, + private readonly func: () => Promise, + private readonly createDate: (dateString?: string) => Date = (s) => + s ? new Date(s) : new Date(), + ) {} + + /** + * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation. + */ + public async invokeFunctionIfIntervalElapsed( + minSecondsSinceLastInvocation: number, + ): Promise> { + const updateCheckStartDate = this.createDate(); + const lastInvocationDate = this.getLastInvocationDate(); + if ( + minSecondsSinceLastInvocation && + lastInvocationDate && + lastInvocationDate <= updateCheckStartDate && + lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 > + updateCheckStartDate.getTime() + ) { + return createRateLimitedResult(); + } + const result = await this.func(); + await this.setLastInvocationDate(updateCheckStartDate); + return createInvokedResult(result); + } + + private getLastInvocationDate(): Date | undefined { + const maybeDateString: string | undefined = this.globalState.get( + InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier, + ); + return maybeDateString ? this.createDate(maybeDateString) : undefined; + } + + private async setLastInvocationDate(date: Date): Promise { + return await this.globalState.update( + InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier, + date, + ); + } + + private static readonly _invocationRateLimiterPrefix = + "invocationRateLimiter_lastInvocationDate_"; +} + +export enum InvocationRateLimiterResultKind { + Invoked, + RateLimited, +} + +/** + * The function was invoked and returned the value `result`. + */ +interface InvokedResult { + kind: InvocationRateLimiterResultKind.Invoked; + result: T; +} + +/** + * The function was not invoked as the minimum interval since the last invocation had not elapsed. + */ +interface RateLimitedResult { + kind: InvocationRateLimiterResultKind.RateLimited; +} + +type InvocationRateLimiterResult = InvokedResult | RateLimitedResult; + +function createInvokedResult(result: T): InvokedResult { + return { + kind: InvocationRateLimiterResultKind.Invoked, + result, + }; +} + +function createRateLimitedResult(): RateLimitedResult { + return { + kind: InvocationRateLimiterResultKind.RateLimited, + }; +} diff --git a/extensions/ql-vscode/src/common/vscode/authentication.ts b/extensions/ql-vscode/src/common/vscode/authentication.ts index 625a31d36..1a7190ec4 100644 --- a/extensions/ql-vscode/src/common/vscode/authentication.ts +++ b/extensions/ql-vscode/src/common/vscode/authentication.ts @@ -3,7 +3,7 @@ import * as Octokit from "@octokit/rest"; import { retry } from "@octokit/plugin-retry"; import { Credentials } from "../authentication"; -const GITHUB_AUTH_PROVIDER_ID = "github"; +export const GITHUB_AUTH_PROVIDER_ID = "github"; // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist, // and 'read:packages' for reading private CodeQL packages. diff --git a/extensions/ql-vscode/src/common/vscode/commands.ts b/extensions/ql-vscode/src/common/vscode/commands.ts index dfd765e17..27f8455e5 100644 --- a/extensions/ql-vscode/src/common/vscode/commands.ts +++ b/extensions/ql-vscode/src/common/vscode/commands.ts @@ -49,7 +49,6 @@ export function registerCommandWithErrorHandling( const errorMessage = redactableError(error)`${ getErrorMessage(e) || e } (${commandId})`; - const errorStack = getErrorStack(e); if (e instanceof UserCancellationException) { // User has cancelled this action manually if (e.silent) { @@ -61,6 +60,7 @@ export function registerCommandWithErrorHandling( } } else { // Include the full stack in the error log only. + const errorStack = getErrorStack(e); const fullMessage = errorStack ? `${errorMessage.fullMessage}\n${errorStack}` : errorMessage.fullMessage; diff --git a/extensions/ql-vscode/src/common/selection-commands.ts b/extensions/ql-vscode/src/common/vscode/selection-commands.ts similarity index 95% rename from extensions/ql-vscode/src/common/selection-commands.ts rename to extensions/ql-vscode/src/common/vscode/selection-commands.ts index 186133969..0d930b8e1 100644 --- a/extensions/ql-vscode/src/common/selection-commands.ts +++ b/extensions/ql-vscode/src/common/vscode/selection-commands.ts @@ -1,9 +1,9 @@ -import { showAndLogErrorMessage } from "../helpers"; +import { showAndLogErrorMessage } from "../../helpers"; import { ExplorerSelectionCommandFunction, TreeViewContextMultiSelectionCommandFunction, TreeViewContextSingleSelectionCommandFunction, -} from "./commands"; +} from "../commands"; // A hack to match types that are not an array, which is useful to help avoid // misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated diff --git a/extensions/ql-vscode/src/common/vscode/vscode-app.ts b/extensions/ql-vscode/src/common/vscode/vscode-app.ts index 986d74c9a..9bc12afef 100644 --- a/extensions/ql-vscode/src/common/vscode/vscode-app.ts +++ b/extensions/ql-vscode/src/common/vscode/vscode-app.ts @@ -39,6 +39,14 @@ export class ExtensionApp implements App { return this.extensionContext.workspaceState; } + public get workspaceFolders(): readonly vscode.WorkspaceFolder[] | undefined { + return vscode.workspace.workspaceFolders; + } + + public get onDidChangeWorkspaceFolders(): vscode.Event { + return vscode.workspace.onDidChangeWorkspaceFolders; + } + public get subscriptions(): Disposable[] { return this.extensionContext.subscriptions; } diff --git a/extensions/ql-vscode/src/compare/compare-view.ts b/extensions/ql-vscode/src/compare/compare-view.ts index df684d19e..c9c033f34 100644 --- a/extensions/ql-vscode/src/compare/compare-view.ts +++ b/extensions/ql-vscode/src/compare/compare-view.ts @@ -175,21 +175,40 @@ export class CompareView extends AbstractWebview< const commonResultSetNames = fromSchemaNames.filter((name) => toSchemaNames.includes(name), ); + + // Fall back on the default result set names if there are no common ones. + const defaultFromResultSetName = fromSchemaNames.find((name) => + name.startsWith("#"), + ); + const defaultToResultSetName = toSchemaNames.find((name) => + name.startsWith("#"), + ); + + if ( + commonResultSetNames.length === 0 && + !(defaultFromResultSetName || defaultToResultSetName) + ) { + throw new Error( + "No common result sets found between the two queries. Please check that the queries are compatible.", + ); + } + const currentResultSetName = selectedResultSetName || commonResultSetNames[0]; const fromResultSet = await this.getResultSet( fromSchemas, - currentResultSetName, + currentResultSetName || defaultFromResultSetName!, from.completedQuery.query.resultsPaths.resultsPath, ); const toResultSet = await this.getResultSet( toSchemas, - currentResultSetName, + currentResultSetName || defaultToResultSetName!, to.completedQuery.query.resultsPaths.resultsPath, ); return [ commonResultSetNames, - currentResultSetName, + currentResultSetName || + `${defaultFromResultSetName} <-> ${defaultToResultSetName}`, fromResultSet, toResultSet, ]; diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts index bb00a1a4b..48c168773 100644 --- a/extensions/ql-vscode/src/config.ts +++ b/extensions/ql-vscode/src/config.ts @@ -711,3 +711,10 @@ const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING); export function showQueriesPanel(): boolean { return !!QUERIES_PANEL.getValue(); } + +const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING); +const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS); + +export function showLlmGeneration(): boolean { + return !!LLM_GENERATION.getValue(); +} diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts new file mode 100644 index 000000000..671c71096 --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts @@ -0,0 +1,54 @@ +import { Credentials } from "../common/authentication"; +import { OctokitResponse } from "@octokit/types"; + +export enum ClassificationType { + Unknown = "CLASSIFICATION_TYPE_UNKNOWN", + Neutral = "CLASSIFICATION_TYPE_NEUTRAL", + Source = "CLASSIFICATION_TYPE_SOURCE", + Sink = "CLASSIFICATION_TYPE_SINK", + Summary = "CLASSIFICATION_TYPE_SUMMARY", +} + +export interface Classification { + type: ClassificationType; + kind: string; + explanation: string; +} + +export interface Method { + package: string; + type: string; + name: string; + signature: string; + usages: string[]; + classification?: Classification; + input?: string; + output?: string; +} + +export interface ModelRequest { + language: string; + candidates: Method[]; + samples: Method[]; +} + +export interface ModelResponse { + language: string; + predicted: Method[]; +} + +export async function autoModel( + credentials: Credentials, + request: ModelRequest, +): Promise { + const octokit = await credentials.getOctokit(); + + const response: OctokitResponse = await octokit.request( + "POST /repos/github/codeql/code-scanning/codeql/auto-model", + { + data: request, + }, + ); + + return response.data; +} diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts new file mode 100644 index 000000000..3a4757017 --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts @@ -0,0 +1,136 @@ +import { CancellationTokenSource } from "vscode"; +import { join } from "path"; +import { runQuery } from "./external-api-usage-query"; +import { CodeQLCliServer } from "../codeql-cli/cli"; +import { QueryRunner } from "../query-server"; +import { DatabaseItem } from "../databases/local-databases"; +import { interpretResultsSarif } from "../query-results"; +import { ProgressCallback } from "../common/vscode/progress"; + +type Options = { + cliServer: CodeQLCliServer; + queryRunner: QueryRunner; + databaseItem: DatabaseItem; + queryStorageDir: string; + + progress: ProgressCallback; +}; + +export type UsageSnippetsBySignature = Record; + +export async function getAutoModelUsages({ + cliServer, + queryRunner, + databaseItem, + queryStorageDir, + progress, +}: Options): Promise { + const maxStep = 1500; + + const cancellationTokenSource = new CancellationTokenSource(); + + // This will re-run the query that was already run when opening the data extensions editor. This + // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we + // need to interpret the results. + const queryResult = await runQuery({ + cliServer, + queryRunner, + queryStorageDir, + databaseItem, + progress: (update) => + progress({ + maxStep, + step: update.step, + message: update.message, + }), + token: cancellationTokenSource.token, + }); + if (!queryResult) { + throw new Error("Query failed"); + } + + progress({ + maxStep, + step: 1100, + message: "Retrieving source location prefix", + }); + + // CodeQL needs to have access to the database to be able to retrieve the + // snippets from it. The source location prefix is used to determine the + // base path of the database. + const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix( + cliServer, + ); + const sourceArchiveUri = databaseItem.sourceArchive; + const sourceInfo = + sourceArchiveUri === undefined + ? undefined + : { + sourceArchive: sourceArchiveUri.fsPath, + sourceLocationPrefix, + }; + + progress({ + maxStep, + step: 1200, + message: "Interpreting results", + }); + + // Convert the results to SARIF so that Codeql will retrieve the snippets + // from the datababe. This means we don't need to do that in the extension + // and everything is handled by the CodeQL CLI. + const sarif = await interpretResultsSarif( + cliServer, + { + // To interpret the results we need to provide metadata about the query. We could do this using + // `resolveMetadata` but that would be an extra call to the CodeQL CLI server and would require + // us to know the path to the query on the filesystem. Since we know what the metadata should + // look like and the only metadata that the CodeQL CLI requires is an ID and the kind, we can + // simply use constants here. + kind: "problem", + id: "usage", + }, + { + resultsPath: queryResult.outputDir.bqrsPath, + interpretedResultsPath: join( + queryStorageDir, + "interpreted-results.sarif", + ), + }, + sourceInfo, + ["--sarif-add-snippets"], + ); + + progress({ + maxStep, + step: 1400, + message: "Parsing results", + }); + + const snippets: UsageSnippetsBySignature = {}; + + const results = sarif.runs[0]?.results; + if (!results) { + throw new Error("No results"); + } + + // This will group the snippets by the method signature. + for (const result of results) { + const signature = result.message.text; + + const snippet = + result.locations?.[0]?.physicalLocation?.contextRegion?.snippet?.text; + + if (!signature || !snippet) { + continue; + } + + if (!(signature in snippets)) { + snippets[signature] = []; + } + + snippets[signature].push(snippet); + } + + return snippets; +} diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts new file mode 100644 index 000000000..af3a6dab1 --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts @@ -0,0 +1,222 @@ +import { ExternalApiUsage } from "./external-api-usage"; +import { ModeledMethod, ModeledMethodType } from "./modeled-method"; +import { + Classification, + ClassificationType, + Method, + ModelRequest, +} from "./auto-model-api"; +import type { UsageSnippetsBySignature } from "./auto-model-usages-query"; + +export function createAutoModelRequest( + language: string, + externalApiUsages: ExternalApiUsage[], + modeledMethods: Record, + usages: UsageSnippetsBySignature, +): ModelRequest { + const request: ModelRequest = { + language, + samples: [], + candidates: [], + }; + + // Sort by number of usages so we always send the most used methods first + externalApiUsages = [...externalApiUsages]; + externalApiUsages.sort((a, b) => b.usages.length - a.usages.length); + + for (const externalApiUsage of externalApiUsages) { + const modeledMethod: ModeledMethod = modeledMethods[ + externalApiUsage.signature + ] ?? { + type: "none", + }; + + const usagesForMethod = + usages[externalApiUsage.signature] ?? + externalApiUsage.usages.map((usage) => usage.label); + + const numberOfArguments = + externalApiUsage.methodParameters === "()" + ? 0 + : externalApiUsage.methodParameters.split(",").length; + + for ( + let argumentIndex = 0; + argumentIndex < numberOfArguments; + argumentIndex++ + ) { + const method: Method = { + package: externalApiUsage.packageName, + type: externalApiUsage.typeName, + name: externalApiUsage.methodName, + signature: externalApiUsage.methodParameters, + classification: + modeledMethod.type === "none" + ? undefined + : toMethodClassification(modeledMethod), + usages: usagesForMethod.slice(0, 10), + input: `Argument[${argumentIndex}]`, + }; + + if (modeledMethod.type === "none") { + request.candidates.push(method); + } else { + request.samples.push(method); + } + } + } + + request.candidates = request.candidates.slice(0, 20); + request.samples = request.samples.slice(0, 100); + + return request; +} + +/** + * For now, we have a simplified model that only models methods as sinks. It does not model methods as neutral, + * so we aren't actually able to correctly determine that a method is neutral; it could still be a source or summary. + * However, to keep this method simple and give output to the user, we will model any method for which none of its + * arguments are modeled as sinks as neutral. + * + * If there are multiple arguments which are modeled as sinks, we will only model the first one. + */ +export function parsePredictedClassifications( + predicted: Method[], +): Record { + const predictedBySignature: Record = {}; + for (const method of predicted) { + if (!method.classification) { + continue; + } + + const signature = toFullMethodSignature(method); + + if (!(signature in predictedBySignature)) { + predictedBySignature[signature] = []; + } + + predictedBySignature[signature].push(method); + } + + const modeledMethods: Record = {}; + + for (const signature in predictedBySignature) { + const predictedMethods = predictedBySignature[signature]; + + const sinks = predictedMethods.filter( + (method) => method.classification?.type === ClassificationType.Sink, + ); + if (sinks.length === 0) { + // For now, model any method for which none of its arguments are modeled as sinks as neutral + modeledMethods[signature] = { + type: "neutral", + kind: "", + input: "", + output: "", + }; + continue; + } + + // Order the sinks by the input alphabetically. This will ensure that the first argument is always + // first in the list of sinks, the second argument is always second, etc. + // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first + sinks.sort((a, b) => compareInputOutput(a.input ?? "", b.input ?? "")); + + const sink = sinks[0]; + + modeledMethods[signature] = { + type: "sink", + kind: sink.classification?.kind ?? "", + input: sink.input ?? "", + output: sink.output ?? "", + }; + } + + return modeledMethods; +} + +function toMethodClassificationType( + type: ModeledMethodType, +): ClassificationType { + switch (type) { + case "source": + return ClassificationType.Source; + case "sink": + return ClassificationType.Sink; + case "summary": + return ClassificationType.Summary; + case "neutral": + return ClassificationType.Neutral; + default: + return ClassificationType.Unknown; + } +} + +function toMethodClassification(modeledMethod: ModeledMethod): Classification { + return { + type: toMethodClassificationType(modeledMethod.type), + kind: modeledMethod.kind, + explanation: "", + }; +} + +function toFullMethodSignature(method: Method): string { + return `${method.package}.${method.type}#${method.name}${method.signature}`; +} + +const argumentRegex = /^Argument\[(\d+)]$/; + +// Argument[this] is before ReturnValue +const nonNumericArgumentOrder = ["Argument[this]", "ReturnValue"]; + +/** + * Compare two inputs or outputs matching `Argument[]`, `Argument[this]`, or `ReturnValue`. + * If they are the same, return 0. If a is less than b, returns a negative number. + * If a is greater than b, returns a positive number. + */ +export function compareInputOutput(a: string, b: string): number { + if (a === b) { + return 0; + } + + const aMatch = a.match(argumentRegex); + const bMatch = b.match(argumentRegex); + + // Numeric arguments are always first + if (aMatch && !bMatch) { + return -1; + } + if (!aMatch && bMatch) { + return 1; + } + + // Neither is an argument + if (!aMatch && !bMatch) { + const aIndex = nonNumericArgumentOrder.indexOf(a); + const bIndex = nonNumericArgumentOrder.indexOf(b); + + // If either one is unknown, it is sorted last + if (aIndex === -1 && bIndex === -1) { + return a.localeCompare(b); + } + if (aIndex === -1) { + return 1; + } + if (bIndex === -1) { + return -1; + } + + return aIndex - bIndex; + } + + // This case shouldn't happen, but makes TypeScript happy + if (!aMatch || !bMatch) { + return 0; + } + + // Both are arguments + const aIndex = parseInt(aMatch[1]); + const bIndex = parseInt(bMatch[1]); + + return aIndex - bIndex; +} diff --git a/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts b/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts index ed1f00c6a..a5c65fe13 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts @@ -7,9 +7,9 @@ export function decodeBqrsToExternalApiUsages( const methodsByApiName = new Map(); chunk?.tuples.forEach((tuple) => { - const signature = tuple[0] as string; - const supported = tuple[1] as boolean; - const usage = tuple[2] as Call; + const usage = tuple[0] as Call; + const signature = tuple[1] as string; + const supported = (tuple[2] as string) === "true"; const [packageWithType, methodDeclaration] = signature.split("#"); diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 676755a69..0ff6e8fcb 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -38,6 +38,13 @@ import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml"; import { ExternalApiUsage } from "./external-api-usage"; import { ModeledMethod } from "./modeled-method"; import { ExtensionPackModelFile } from "./shared/extension-pack"; +import { autoModel } from "./auto-model-api"; +import { + createAutoModelRequest, + parsePredictedClassifications, +} from "./auto-model"; +import { showLlmGeneration } from "../config"; +import { getAutoModelUsages } from "./auto-model-usages-query"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, @@ -113,6 +120,13 @@ export class DataExtensionsEditorView extends AbstractWebview< case "generateExternalApi": await this.generateModeledMethods(); + break; + case "generateExternalApiFromLlm": + await this.generateModeledMethodsFromLlm( + msg.externalApiUsages, + msg.modeledMethods, + ); + break; default: assertNever(msg); @@ -135,6 +149,7 @@ export class DataExtensionsEditorView extends AbstractWebview< viewState: { extensionPackModelFile: this.modelFile, modelFileExists: await pathExists(this.modelFile.filename), + showLlmButton: showLlmGeneration(), }, }); } @@ -347,6 +362,72 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.clearProgress(); } + private async generateModeledMethodsFromLlm( + externalApiUsages: ExternalApiUsage[], + modeledMethods: Record, + ): Promise { + const maxStep = 3000; + + await this.showProgress({ + step: 0, + maxStep, + message: "Retrieving usages", + }); + + const usages = await getAutoModelUsages({ + cliServer: this.cliServer, + queryRunner: this.queryRunner, + queryStorageDir: this.queryStorageDir, + databaseItem: this.databaseItem, + progress: (update) => this.showProgress(update, maxStep), + }); + + await this.showProgress({ + step: 1800, + maxStep, + message: "Creating request", + }); + + const request = createAutoModelRequest( + this.databaseItem.language, + externalApiUsages, + modeledMethods, + usages, + ); + + await this.showProgress({ + step: 2000, + maxStep, + message: "Sending request", + }); + + const response = await autoModel(this.app.credentials, request); + + await this.showProgress({ + step: 2500, + maxStep, + message: "Parsing response", + }); + + const predictedModeledMethods = parsePredictedClassifications( + response.predicted, + ); + + await this.showProgress({ + step: 2800, + maxStep, + message: "Applying results", + }); + + await this.postMessage({ + t: "addModeledMethods", + modeledMethods: predictedModeledMethods, + overrideNone: true, + }); + + await this.clearProgress(); + } + /* * Progress in this class is a bit weird. Most of the progress is based on running the query. * Query progress is always between 0 and 1000. However, we still have some steps that need diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index 9fbc7b293..659f8d519 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -78,7 +78,11 @@ export async function runQuery({ const queryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, - { queryPath: queryFile, quickEvalPosition: undefined }, + { + queryPath: queryFile, + quickEvalPosition: undefined, + quickEvalCountOnly: false, + }, false, getOnDiskWorkspaceFolders(), extensionPacks, diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index d047e7876..7af3ae940 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -92,7 +92,11 @@ async function getModeledMethodsFromFlow( const queryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, - { queryPath, quickEvalPosition: undefined }, + { + queryPath, + quickEvalPosition: undefined, + quickEvalCountOnly: false, + }, false, getOnDiskWorkspaceFolders(), undefined, diff --git a/extensions/ql-vscode/src/data-extensions-editor/predicates.ts b/extensions/ql-vscode/src/data-extensions-editor/predicates.ts index f52115163..65a728173 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/predicates.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/predicates.ts @@ -116,13 +116,14 @@ export const extensiblePredicateDefinitions: Record< neutral: { extensiblePredicate: "neutralModel", // extensible predicate neutralModel( - // string package, string type, string name, string signature, string provenance + // string package, string type, string name, string signature, string kind, string provenance // ); generateMethodDefinition: (method) => [ method.externalApiUsage.packageName, method.externalApiUsage.typeName, method.externalApiUsage.methodName, method.externalApiUsage.methodParameters, + method.modeledMethod.kind, "manual", ], readModeledMethod: (row) => ({ @@ -131,8 +132,9 @@ export const extensiblePredicateDefinitions: Record< type: "neutral", input: "", output: "", - kind: "", + kind: row[4] as string, }, }), + supportedKinds: ["summary", "source", "sink"], }, }; diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts index 5dc25a9eb..ff3757fb3 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts @@ -5,30 +5,28 @@ export const fetchExternalApisQuery: Query = { * @name Usage of APIs coming from external libraries * @description A list of 3rd party APIs used in the codebase. * @tags telemetry + * @kind problem * @id cs/telemetry/fetch-external-apis */ - import csharp - import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl - import ExternalApi - - private Call aUsage(ExternalApi api) { - result.getTarget().getUnboundDeclaration() = api - } - - private boolean isSupported(ExternalApi api) { - api.isSupported() and result = true - or - not api.isSupported() and - result = false - } - - from ExternalApi api, string apiName, boolean supported, Call usage - where - apiName = api.getApiName() and - supported = isSupported(api) and - usage = aUsage(api) - select apiName, supported, usage +import csharp +import ExternalApi + +private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api } + +private boolean isSupported(ExternalApi api) { + api.isSupported() and result = true + or + not api.isSupported() and + result = false +} + +from ExternalApi api, string apiName, boolean supported, Call usage +where + apiName = api.getApiName() and + supported = isSupported(api) and + usage = aUsage(api) +select usage, apiName, supported.toString(), "supported" `, dependencies: { "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */ diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts index 721673e40..0d4010721 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts @@ -5,11 +5,11 @@ export const fetchExternalApisQuery: Query = { * @name Usage of APIs coming from external libraries * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code. * @tags telemetry + * @kind problem * @id java/telemetry/fetch-external-apis */ import java -import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl import ExternalApi private Call aUsage(ExternalApi api) { @@ -28,7 +28,7 @@ where apiName = api.getApiName() and supported = isSupported(api) and usage = aUsage(api) -select apiName, supported, usage +select usage, apiName, supported.toString(), "supported" `, dependencies: { "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */ diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts index 72f239529..b09b15e14 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts @@ -1,4 +1,13 @@ export type Query = { + /** + * The main query. + * + * It should select all usages of external APIs, and return the following result pattern: + * - usage: the usage of the external API. This is an entity. + * - apiName: the name of the external API. This is a string. + * - supported: whether the external API is supported by the extension. This should be a string representation of a boolean to satify the result pattern for a problem query. + * - "supported": a string literal. This is required to make the query a valid problem query. + */ mainQuery: string; dependencies?: { [filename: string]: string; diff --git a/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts b/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts index 0da3f0d25..ece8af174 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts @@ -3,4 +3,5 @@ import { ExtensionPackModelFile } from "./extension-pack"; export interface DataExtensionEditorViewState { extensionPackModelFile: ExtensionPackModelFile; modelFileExists: boolean; + showLlmButton: boolean; } diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts index 3013d164f..dc94f1685 100644 --- a/extensions/ql-vscode/src/databases/config/db-config-store.ts +++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts @@ -61,7 +61,9 @@ export class DbConfigStore extends DisposableObject { this.configErrors = []; this.configWatcher = undefined; this.configValidator = new DbConfigValidator(app.extensionPath); - this.onDidChangeConfigEventEmitter = app.createEventEmitter(); + this.onDidChangeConfigEventEmitter = this.push( + app.createEventEmitter(), + ); this.onDidChangeConfig = this.onDidChangeConfigEventEmitter.event; } @@ -145,10 +147,46 @@ export class DbConfigStore extends DisposableObject { await this.writeConfig(config); } + /** + * Adds a list of remote repositories to an existing repository list and removes duplicates. + * @returns a list of repositories that were not added because the list reached 1000 entries. + */ + public async addRemoteReposToList( + repoNwoList: string[], + parentList: string, + ): Promise { + if (!this.config) { + throw Error("Cannot add variant analysis repos if config is not loaded"); + } + + const config = cloneDbConfig(this.config); + const parent = config.databases.variantAnalysis.repositoryLists.find( + (list) => list.name === parentList, + ); + if (!parent) { + throw Error(`Cannot find parent list '${parentList}'`); + } + + // Remove duplicates from the list of repositories. + const newRepositoriesList = [ + ...new Set([...parent.repositories, ...repoNwoList]), + ]; + + parent.repositories = newRepositoriesList.slice(0, 1000); + const truncatedRepositories = newRepositoriesList.slice(1000); + + await this.writeConfig(config); + return truncatedRepositories; + } + + /** + * Adds one remote repository + * @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added. + */ public async addRemoteRepo( repoNwo: string, parentList?: string, - ): Promise { + ): Promise { if (!this.config) { throw Error("Cannot add variant analysis repo if config is not loaded"); } @@ -163,6 +201,7 @@ export class DbConfigStore extends DisposableObject { ); } + const truncatedRepositories = []; const config = cloneDbConfig(this.config); if (parentList) { const parent = config.databases.variantAnalysis.repositoryLists.find( @@ -171,12 +210,15 @@ export class DbConfigStore extends DisposableObject { if (!parent) { throw Error(`Cannot find parent list '${parentList}'`); } else { - parent.repositories.push(repoNwo); + const newRepositories = [...parent.repositories, repoNwo]; + parent.repositories = newRepositories.slice(0, 1000); + truncatedRepositories.push(...newRepositories.slice(1000)); } } else { config.databases.variantAnalysis.repositories.push(repoNwo); } await this.writeConfig(config); + return truncatedRepositories; } public async addRemoteOwner(owner: string): Promise { diff --git a/extensions/ql-vscode/src/databases/db-manager.ts b/extensions/ql-vscode/src/databases/db-manager.ts index cc85bccc0..387b35a8e 100644 --- a/extensions/ql-vscode/src/databases/db-manager.ts +++ b/extensions/ql-vscode/src/databases/db-manager.ts @@ -1,6 +1,7 @@ import { App } from "../common/app"; import { AppEvent, AppEventEmitter } from "../common/events"; import { ValueResult } from "../common/value-result"; +import { DisposableObject } from "../pure/disposable-object"; import { DbConfigStore } from "./config/db-config-store"; import { DbItem, @@ -23,7 +24,7 @@ import { import { createRemoteTree } from "./db-tree-creator"; import { DbConfigValidationError } from "./db-validation-errors"; -export class DbManager { +export class DbManager extends DisposableObject { public readonly onDbItemsChanged: AppEvent; public static readonly DB_EXPANDED_STATE_KEY = "db_expanded"; private readonly onDbItemsChangesEventEmitter: AppEventEmitter; @@ -32,7 +33,11 @@ export class DbManager { private readonly app: App, private readonly dbConfigStore: DbConfigStore, ) { - this.onDbItemsChangesEventEmitter = app.createEventEmitter(); + super(); + + this.onDbItemsChangesEventEmitter = this.push( + app.createEventEmitter(), + ); this.onDbItemsChanged = this.onDbItemsChangesEventEmitter.event; this.dbConfigStore.onDidChangeConfig(() => { @@ -96,8 +101,15 @@ export class DbManager { public async addNewRemoteRepo( nwo: string, parentList?: string, - ): Promise { - await this.dbConfigStore.addRemoteRepo(nwo, parentList); + ): Promise { + return await this.dbConfigStore.addRemoteRepo(nwo, parentList); + } + + public async addNewRemoteReposToList( + nwoList: string[], + parentList: string, + ): Promise { + return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList); } public async addNewRemoteOwner(owner: string): Promise { diff --git a/extensions/ql-vscode/src/databases/db-module.ts b/extensions/ql-vscode/src/databases/db-module.ts index 8d47f75a8..0ca2898db 100644 --- a/extensions/ql-vscode/src/databases/db-module.ts +++ b/extensions/ql-vscode/src/databases/db-module.ts @@ -17,7 +17,7 @@ export class DbModule extends DisposableObject { super(); this.dbConfigStore = new DbConfigStore(app); - this.dbManager = new DbManager(app, this.dbConfigStore); + this.dbManager = this.push(new DbManager(app, this.dbConfigStore)); } public static async initialize(app: App): Promise { diff --git a/extensions/ql-vscode/src/databases/local-databases-ui.ts b/extensions/ql-vscode/src/databases/local-databases-ui.ts index 0839b2afc..5fbc23c56 100644 --- a/extensions/ql-vscode/src/databases/local-databases-ui.ts +++ b/extensions/ql-vscode/src/databases/local-databases-ui.ts @@ -49,7 +49,7 @@ import { LocalDatabasesCommands } from "../common/commands"; import { createMultiSelectionCommand, createSingleSelectionCommand, -} from "../common/selection-commands"; +} from "../common/vscode/selection-commands"; enum SortOrder { NameAsc = "NameAsc", diff --git a/extensions/ql-vscode/src/databases/local-databases/database-contents.ts b/extensions/ql-vscode/src/databases/local-databases/database-contents.ts new file mode 100644 index 000000000..ce9f5d760 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-contents.ts @@ -0,0 +1,30 @@ +import vscode from "vscode"; + +/** + * The layout of the database. + */ +export enum DatabaseKind { + /** A CodeQL database */ + Database, + /** A raw QL dataset */ + RawDataset, +} + +export interface DatabaseContents { + /** The layout of the database */ + kind: DatabaseKind; + /** + * The name of the database. + */ + name: string; + /** The URI of the QL dataset within the database. */ + datasetUri: vscode.Uri; + /** The URI of the source archive within the database, if one exists. */ + sourceArchiveUri?: vscode.Uri; + /** The URI of the CodeQL database scheme within the database, if exactly one exists. */ + dbSchemeUri?: vscode.Uri; +} + +export interface DatabaseContentsWithDbScheme extends DatabaseContents { + dbSchemeUri: vscode.Uri; // Always present +} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-events.ts b/extensions/ql-vscode/src/databases/local-databases/database-events.ts new file mode 100644 index 000000000..a48766ffb --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-events.ts @@ -0,0 +1,19 @@ +import { DatabaseItem } from "./database-item"; + +export enum DatabaseEventKind { + Add = "Add", + Remove = "Remove", + + // Fired when databases are refreshed from persisted state + Refresh = "Refresh", + + // Fired when the current database changes + Change = "Change", + + Rename = "Rename", +} + +export interface DatabaseChangedEvent { + kind: DatabaseEventKind; + item: DatabaseItem | undefined; +} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts new file mode 100644 index 000000000..16f184324 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -0,0 +1,213 @@ +// Exported for testing +import * as cli from "../../codeql-cli/cli"; +import vscode from "vscode"; +import { FullDatabaseOptions } from "./database-options"; +import { basename, dirname, join, relative } from "path"; +import { + decodeSourceArchiveUri, + encodeArchiveBasePath, + encodeSourceArchiveUri, + zipArchiveScheme, +} from "../../common/vscode/archive-filesystem-provider"; +import { DatabaseItem, PersistedDatabaseItem } from "./database-item"; +import { isLikelyDatabaseRoot } from "../../helpers"; +import { stat } from "fs-extra"; +import { pathsEqual } from "../../pure/files"; +import { DatabaseContents } from "./database-contents"; + +export class DatabaseItemImpl implements DatabaseItem { + // These are only public in the implementation, they are readonly in the interface + public error: Error | undefined = undefined; + public contents: DatabaseContents | undefined; + /** A cache of database info */ + private _dbinfo: cli.DbInfo | undefined; + + public constructor( + public readonly databaseUri: vscode.Uri, + contents: DatabaseContents | undefined, + private options: FullDatabaseOptions, + ) { + this.contents = contents; + } + + public get name(): string { + if (this.options.displayName) { + return this.options.displayName; + } else if (this.contents) { + return this.contents.name; + } else { + return basename(this.databaseUri.fsPath); + } + } + + public set name(newName: string) { + this.options.displayName = newName; + } + + public get sourceArchive(): vscode.Uri | undefined { + if (this.options.ignoreSourceArchive || this.contents === undefined) { + return undefined; + } else { + return this.contents.sourceArchiveUri; + } + } + + public get dateAdded(): number | undefined { + return this.options.dateAdded; + } + + public resolveSourceFile(uriStr: string | undefined): vscode.Uri { + const sourceArchive = this.sourceArchive; + const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined; + if (uri && uri.scheme !== "file") { + throw new Error( + `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`, + ); + } + if (!sourceArchive) { + if (uri) { + return uri; + } else { + return this.databaseUri; + } + } + + if (uri) { + const relativeFilePath = decodeURI(uri.path) + .replace(":", "_") + .replace(/^\/*/, ""); + if (sourceArchive.scheme === zipArchiveScheme) { + const zipRef = decodeSourceArchiveUri(sourceArchive); + const pathWithinSourceArchive = + zipRef.pathWithinSourceArchive === "/" + ? relativeFilePath + : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`; + return encodeSourceArchiveUri({ + pathWithinSourceArchive, + sourceArchiveZipPath: zipRef.sourceArchiveZipPath, + }); + } else { + let newPath = sourceArchive.path; + if (!newPath.endsWith("/")) { + // Ensure a trailing slash. + newPath += "/"; + } + newPath += relativeFilePath; + + return sourceArchive.with({ path: newPath }); + } + } else { + return sourceArchive; + } + } + + /** + * Gets the state of this database, to be persisted in the workspace state. + */ + public getPersistedState(): PersistedDatabaseItem { + return { + uri: this.databaseUri.toString(true), + options: this.options, + }; + } + + /** + * Holds if the database item refers to an exported snapshot + */ + public async hasMetadataFile(): Promise { + return await isLikelyDatabaseRoot(this.databaseUri.fsPath); + } + + /** + * Returns information about a database. + */ + private async getDbInfo(server: cli.CodeQLCliServer): Promise { + if (this._dbinfo === undefined) { + this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath); + } + return this._dbinfo; + } + + /** + * Returns `sourceLocationPrefix` of database. Requires that the database + * has a `.dbinfo` file, which is the source of the prefix. + */ + public async getSourceLocationPrefix( + server: cli.CodeQLCliServer, + ): Promise { + const dbInfo = await this.getDbInfo(server); + return dbInfo.sourceLocationPrefix; + } + + /** + * Returns path to dataset folder of database. + */ + public async getDatasetFolder(server: cli.CodeQLCliServer): Promise { + const dbInfo = await this.getDbInfo(server); + return dbInfo.datasetFolder; + } + + public get language() { + return this.options.language || ""; + } + + /** + * Returns the root uri of the virtual filesystem for this database's source archive. + */ + public getSourceArchiveExplorerUri(): vscode.Uri { + const sourceArchive = this.sourceArchive; + if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) { + throw new Error(this.verifyZippedSources()); + } + return encodeArchiveBasePath(sourceArchive.fsPath); + } + + public verifyZippedSources(): string | undefined { + const sourceArchive = this.sourceArchive; + if (sourceArchive === undefined) { + return `${this.name} has no source archive.`; + } + + if (!sourceArchive.fsPath.endsWith(".zip")) { + return `${this.name} has a source folder that is unzipped.`; + } + return; + } + + /** + * Holds if `uri` belongs to this database's source archive. + */ + public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean { + if (this.sourceArchive === undefined) return false; + return ( + uri.scheme === zipArchiveScheme && + decodeSourceArchiveUri(uri).sourceArchiveZipPath === + this.sourceArchive.fsPath + ); + } + + public async isAffectedByTest(testPath: string): Promise { + const databasePath = this.databaseUri.fsPath; + if (!databasePath.endsWith(".testproj")) { + return false; + } + try { + const stats = await stat(testPath); + if (stats.isDirectory()) { + return !relative(testPath, databasePath).startsWith(".."); + } else { + // database for /one/two/three/test.ql is at /one/two/three/three.testproj + const testdir = dirname(testPath); + const testdirbase = basename(testdir); + return pathsEqual( + databasePath, + join(testdir, `${testdirbase}.testproj`), + process.platform, + ); + } + } catch { + // No information available for test path - assume database is unaffected. + return false; + } + } +} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts new file mode 100644 index 000000000..1794d8e75 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts @@ -0,0 +1,83 @@ +import vscode from "vscode"; +import * as cli from "../../codeql-cli/cli"; +import { DatabaseContents } from "./database-contents"; +import { DatabaseOptions } from "./database-options"; + +/** An item in the list of available databases */ +export interface DatabaseItem { + /** The URI of the database */ + readonly databaseUri: vscode.Uri; + /** The name of the database to be displayed in the UI */ + name: string; + + /** The primary language of the database or empty string if unknown */ + readonly language: string; + /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */ + readonly sourceArchive: vscode.Uri | undefined; + /** + * The contents of the database. + * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`. + */ + readonly contents: DatabaseContents | undefined; + + /** + * The date this database was added as a unix timestamp. Or undefined if we don't know. + */ + readonly dateAdded: number | undefined; + + /** If the database is invalid, describes why. */ + readonly error: Error | undefined; + + /** + * Resolves a filename to its URI in the source archive. + * + * @param file Filename within the source archive. May be `undefined` to return a dummy file path. + */ + resolveSourceFile(file: string | undefined): vscode.Uri; + + /** + * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file. + */ + hasMetadataFile(): Promise; + + /** + * Returns `sourceLocationPrefix` of exported database. + */ + getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise; + + /** + * Returns dataset folder of exported database. + */ + getDatasetFolder(server: cli.CodeQLCliServer): Promise; + + /** + * Returns the root uri of the virtual filesystem for this database's source archive, + * as displayed in the filesystem explorer. + */ + getSourceArchiveExplorerUri(): vscode.Uri; + + /** + * Holds if `uri` belongs to this database's source archive. + */ + belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean; + + /** + * Whether the database may be affected by test execution for the given path. + */ + isAffectedByTest(testPath: string): Promise; + + /** + * Gets the state of this database, to be persisted in the workspace state. + */ + getPersistedState(): PersistedDatabaseItem; + + /** + * Verifies that this database item has a zipped source folder. Returns an error message if it does not. + */ + verifyZippedSources(): string | undefined; +} + +export interface PersistedDatabaseItem { + uri: string; + options?: DatabaseOptions; +} diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts similarity index 53% rename from extensions/ql-vscode/src/databases/local-databases.ts rename to extensions/ql-vscode/src/databases/local-databases/database-manager.ts index 8bc50c70f..726ab554a 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -1,50 +1,34 @@ -import { pathExists, stat, remove } from "fs-extra"; -import { glob } from "glob"; -import { join, basename, resolve, relative, dirname, extname } from "path"; -import * as vscode from "vscode"; -import * as cli from "../codeql-cli/cli"; -import { ExtensionContext } from "vscode"; -import { - showAndLogWarningMessage, - showAndLogInformationMessage, - isLikelyDatabaseRoot, - showAndLogExceptionWithTelemetry, - isFolderAlreadyInWorkspace, - getFirstWorkspaceFolder, - showNeverAskAgainDialog, -} from "../helpers"; -import { ProgressCallback, withProgress } from "../common/vscode/progress"; -import { - zipArchiveScheme, - encodeArchiveBasePath, - decodeSourceArchiveUri, - encodeSourceArchiveUri, -} from "../common/vscode/archive-filesystem-provider"; -import { DisposableObject } from "../pure/disposable-object"; -import { Logger, extLogger } from "../common"; -import { asError, getErrorMessage } from "../pure/helpers-pure"; -import { QueryRunner } from "../query-server"; -import { containsPath, pathsEqual } from "../pure/files"; -import { redactableError } from "../pure/errors"; +import vscode, { ExtensionContext } from "vscode"; +import { extLogger, Logger } from "../../common"; +import { DisposableObject } from "../../pure/disposable-object"; +import { App } from "../../common/app"; +import { QueryRunner } from "../../query-server"; +import * as cli from "../../codeql-cli/cli"; +import { ProgressCallback, withProgress } from "../../common/vscode/progress"; import { getAutogenerateQlPacks, isCodespacesTemplate, setAutogenerateQlPacks, -} from "../config"; -import { QlPackGenerator } from "../qlpack-generator"; -import { QueryLanguage } from "../common/query-language"; -import { App } from "../common/app"; +} from "../../config"; +import { extname, join } from "path"; +import { FullDatabaseOptions } from "./database-options"; +import { DatabaseItemImpl } from "./database-item-impl"; +import { + getFirstWorkspaceFolder, + isFolderAlreadyInWorkspace, + showAndLogExceptionWithTelemetry, + showNeverAskAgainDialog, +} from "../../helpers"; import { existsSync } from "fs"; - -/** - * databases.ts - * ------------ - * Managing state of what the current database is, and what other - * databases have been recently selected. - * - * The source of truth of the current state resides inside the - * `DatabaseManager` class below. - */ +import { QlPackGenerator } from "../../qlpack-generator"; +import { QueryLanguage } from "../../common/query-language"; +import { asError, getErrorMessage } from "../../pure/helpers-pure"; +import { DatabaseItem, PersistedDatabaseItem } from "./database-item"; +import { redactableError } from "../../pure/errors"; +import { remove } from "fs-extra"; +import { containsPath } from "../../pure/files"; +import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; +import { DatabaseResolver } from "./database-resolver"; /** * The name of the key in the workspaceState dictionary in which we @@ -58,509 +42,6 @@ const CURRENT_DB = "currentDatabase"; */ const DB_LIST = "databaseList"; -export interface DatabaseOptions { - displayName?: string; - ignoreSourceArchive?: boolean; - dateAdded?: number | undefined; - language?: string; -} - -export interface FullDatabaseOptions extends DatabaseOptions { - ignoreSourceArchive: boolean; - dateAdded: number | undefined; - language: string | undefined; -} - -interface PersistedDatabaseItem { - uri: string; - options?: DatabaseOptions; -} - -/** - * The layout of the database. - */ -export enum DatabaseKind { - /** A CodeQL database */ - Database, - /** A raw QL dataset */ - RawDataset, -} - -export interface DatabaseContents { - /** The layout of the database */ - kind: DatabaseKind; - /** - * The name of the database. - */ - name: string; - /** The URI of the QL dataset within the database. */ - datasetUri: vscode.Uri; - /** The URI of the source archive within the database, if one exists. */ - sourceArchiveUri?: vscode.Uri; - /** The URI of the CodeQL database scheme within the database, if exactly one exists. */ - dbSchemeUri?: vscode.Uri; -} - -export interface DatabaseContentsWithDbScheme extends DatabaseContents { - dbSchemeUri: vscode.Uri; // Always present -} - -/** - * An error thrown when we cannot find a valid database in a putative - * database directory. - */ -class InvalidDatabaseError extends Error {} - -async function findDataset(parentDirectory: string): Promise { - /* - * Look directly in the root - */ - let dbRelativePaths = await glob("db-*/", { - cwd: parentDirectory, - }); - - if (dbRelativePaths.length === 0) { - /* - * Check If they are in the old location - */ - dbRelativePaths = await glob("working/db-*/", { - cwd: parentDirectory, - }); - } - if (dbRelativePaths.length === 0) { - throw new InvalidDatabaseError( - `'${parentDirectory}' does not contain a dataset directory.`, - ); - } - - const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]); - if (dbRelativePaths.length > 1) { - void showAndLogWarningMessage( - `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`, - ); - } - - return vscode.Uri.file(dbAbsolutePath); -} - -// exported for testing -export async function findSourceArchive( - databasePath: string, -): Promise { - const relativePaths = ["src", "output/src_archive"]; - - for (const relativePath of relativePaths) { - const basePath = join(databasePath, relativePath); - const zipPath = `${basePath}.zip`; - - // Prefer using a zip archive over a directory. - if (await pathExists(zipPath)) { - return encodeArchiveBasePath(zipPath); - } else if (await pathExists(basePath)) { - return vscode.Uri.file(basePath); - } - } - - void showAndLogInformationMessage( - `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`, - ); - return undefined; -} - -/** Gets the relative paths of all `.dbscheme` files in the given directory. */ -async function getDbSchemeFiles(dbDirectory: string): Promise { - return await glob("*.dbscheme", { cwd: dbDirectory }); -} - -export class DatabaseResolver { - public static async resolveDatabaseContents( - uri: vscode.Uri, - ): Promise { - if (uri.scheme !== "file") { - throw new Error( - `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`, - ); - } - const databasePath = uri.fsPath; - if (!(await pathExists(databasePath))) { - throw new InvalidDatabaseError( - `Database '${databasePath}' does not exist.`, - ); - } - - const contents = await this.resolveDatabase(databasePath); - - if (contents === undefined) { - throw new InvalidDatabaseError( - `'${databasePath}' is not a valid database.`, - ); - } - - // Look for a single dbscheme file within the database. - // This should be found in the dataset directory, regardless of the form of database. - const dbPath = contents.datasetUri.fsPath; - const dbSchemeFiles = await getDbSchemeFiles(dbPath); - if (dbSchemeFiles.length === 0) { - throw new InvalidDatabaseError( - `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`, - ); - } else if (dbSchemeFiles.length > 1) { - throw new InvalidDatabaseError( - `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`, - ); - } else { - const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0])); - return { - ...contents, - dbSchemeUri, - }; - } - } - - public static async resolveDatabase( - databasePath: string, - ): Promise { - const name = basename(databasePath); - - // Look for dataset and source archive. - const datasetUri = await findDataset(databasePath); - const sourceArchiveUri = await findSourceArchive(databasePath); - - return { - kind: DatabaseKind.Database, - name, - datasetUri, - sourceArchiveUri, - }; - } -} - -/** An item in the list of available databases */ -export interface DatabaseItem { - /** The URI of the database */ - readonly databaseUri: vscode.Uri; - /** The name of the database to be displayed in the UI */ - name: string; - - /** The primary language of the database or empty string if unknown */ - readonly language: string; - /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */ - readonly sourceArchive: vscode.Uri | undefined; - /** - * The contents of the database. - * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`. - */ - readonly contents: DatabaseContents | undefined; - - /** - * The date this database was added as a unix timestamp. Or undefined if we don't know. - */ - readonly dateAdded: number | undefined; - - /** If the database is invalid, describes why. */ - readonly error: Error | undefined; - /** - * Resolves the contents of the database. - * - * @remarks - * The contents include the database directory, source archive, and metadata about the database. - * If the database is invalid, `this.error` is updated with the error object that describes why - * the database is invalid. This error is also thrown. - */ - refresh(): Promise; - /** - * Resolves a filename to its URI in the source archive. - * - * @param file Filename within the source archive. May be `undefined` to return a dummy file path. - */ - resolveSourceFile(file: string | undefined): vscode.Uri; - - /** - * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file. - */ - hasMetadataFile(): Promise; - - /** - * Returns `sourceLocationPrefix` of exported database. - */ - getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise; - - /** - * Returns dataset folder of exported database. - */ - getDatasetFolder(server: cli.CodeQLCliServer): Promise; - - /** - * Returns the root uri of the virtual filesystem for this database's source archive, - * as displayed in the filesystem explorer. - */ - getSourceArchiveExplorerUri(): vscode.Uri; - - /** - * Holds if `uri` belongs to this database's source archive. - */ - belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean; - - /** - * Whether the database may be affected by test execution for the given path. - */ - isAffectedByTest(testPath: string): Promise; - - /** - * Gets the state of this database, to be persisted in the workspace state. - */ - getPersistedState(): PersistedDatabaseItem; - - /** - * Verifies that this database item has a zipped source folder. Returns an error message if it does not. - */ - verifyZippedSources(): string | undefined; -} - -export enum DatabaseEventKind { - Add = "Add", - Remove = "Remove", - - // Fired when databases are refreshed from persisted state - Refresh = "Refresh", - - // Fired when the current database changes - Change = "Change", - - Rename = "Rename", -} - -export interface DatabaseChangedEvent { - kind: DatabaseEventKind; - item: DatabaseItem | undefined; -} - -// Exported for testing -export class DatabaseItemImpl implements DatabaseItem { - private _error: Error | undefined = undefined; - private _contents: DatabaseContents | undefined; - /** A cache of database info */ - private _dbinfo: cli.DbInfo | undefined; - - public constructor( - public readonly databaseUri: vscode.Uri, - contents: DatabaseContents | undefined, - private options: FullDatabaseOptions, - private readonly onChanged: (event: DatabaseChangedEvent) => void, - ) { - this._contents = contents; - } - - public get name(): string { - if (this.options.displayName) { - return this.options.displayName; - } else if (this._contents) { - return this._contents.name; - } else { - return basename(this.databaseUri.fsPath); - } - } - - public set name(newName: string) { - this.options.displayName = newName; - } - - public get sourceArchive(): vscode.Uri | undefined { - if (this.options.ignoreSourceArchive || this._contents === undefined) { - return undefined; - } else { - return this._contents.sourceArchiveUri; - } - } - - public get contents(): DatabaseContents | undefined { - return this._contents; - } - - public get dateAdded(): number | undefined { - return this.options.dateAdded; - } - - public get error(): Error | undefined { - return this._error; - } - - public async refresh(): Promise { - try { - try { - this._contents = await DatabaseResolver.resolveDatabaseContents( - this.databaseUri, - ); - this._error = undefined; - } catch (e) { - this._contents = undefined; - this._error = asError(e); - throw e; - } - } finally { - this.onChanged({ - kind: DatabaseEventKind.Refresh, - item: this, - }); - } - } - - public resolveSourceFile(uriStr: string | undefined): vscode.Uri { - const sourceArchive = this.sourceArchive; - const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined; - if (uri && uri.scheme !== "file") { - throw new Error( - `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`, - ); - } - if (!sourceArchive) { - if (uri) { - return uri; - } else { - return this.databaseUri; - } - } - - if (uri) { - const relativeFilePath = decodeURI(uri.path) - .replace(":", "_") - .replace(/^\/*/, ""); - if (sourceArchive.scheme === zipArchiveScheme) { - const zipRef = decodeSourceArchiveUri(sourceArchive); - const pathWithinSourceArchive = - zipRef.pathWithinSourceArchive === "/" - ? relativeFilePath - : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`; - return encodeSourceArchiveUri({ - pathWithinSourceArchive, - sourceArchiveZipPath: zipRef.sourceArchiveZipPath, - }); - } else { - let newPath = sourceArchive.path; - if (!newPath.endsWith("/")) { - // Ensure a trailing slash. - newPath += "/"; - } - newPath += relativeFilePath; - - return sourceArchive.with({ path: newPath }); - } - } else { - return sourceArchive; - } - } - - /** - * Gets the state of this database, to be persisted in the workspace state. - */ - public getPersistedState(): PersistedDatabaseItem { - return { - uri: this.databaseUri.toString(true), - options: this.options, - }; - } - - /** - * Holds if the database item refers to an exported snapshot - */ - public async hasMetadataFile(): Promise { - return await isLikelyDatabaseRoot(this.databaseUri.fsPath); - } - - /** - * Returns information about a database. - */ - private async getDbInfo(server: cli.CodeQLCliServer): Promise { - if (this._dbinfo === undefined) { - this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath); - } - return this._dbinfo; - } - - /** - * Returns `sourceLocationPrefix` of database. Requires that the database - * has a `.dbinfo` file, which is the source of the prefix. - */ - public async getSourceLocationPrefix( - server: cli.CodeQLCliServer, - ): Promise { - const dbInfo = await this.getDbInfo(server); - return dbInfo.sourceLocationPrefix; - } - - /** - * Returns path to dataset folder of database. - */ - public async getDatasetFolder(server: cli.CodeQLCliServer): Promise { - const dbInfo = await this.getDbInfo(server); - return dbInfo.datasetFolder; - } - - public get language() { - return this.options.language || ""; - } - - /** - * Returns the root uri of the virtual filesystem for this database's source archive. - */ - public getSourceArchiveExplorerUri(): vscode.Uri { - const sourceArchive = this.sourceArchive; - if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) { - throw new Error(this.verifyZippedSources()); - } - return encodeArchiveBasePath(sourceArchive.fsPath); - } - - public verifyZippedSources(): string | undefined { - const sourceArchive = this.sourceArchive; - if (sourceArchive === undefined) { - return `${this.name} has no source archive.`; - } - - if (!sourceArchive.fsPath.endsWith(".zip")) { - return `${this.name} has a source folder that is unzipped.`; - } - return; - } - - /** - * Holds if `uri` belongs to this database's source archive. - */ - public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean { - if (this.sourceArchive === undefined) return false; - return ( - uri.scheme === zipArchiveScheme && - decodeSourceArchiveUri(uri).sourceArchiveZipPath === - this.sourceArchive.fsPath - ); - } - - public async isAffectedByTest(testPath: string): Promise { - const databasePath = this.databaseUri.fsPath; - if (!databasePath.endsWith(".testproj")) { - return false; - } - try { - const stats = await stat(testPath); - if (stats.isDirectory()) { - return !relative(testPath, databasePath).startsWith(".."); - } else { - // database for /one/two/three/test.ql is at /one/two/three/three.testproj - const testdir = dirname(testPath); - const testdirbase = basename(testdir); - return pathsEqual( - databasePath, - join(testdir, `${testdirbase}.testproj`), - process.platform, - ); - } - } catch { - // No information available for test path - assume database is unaffected. - return false; - } - } -} - /** * A promise that resolves to an event's result value when the event * `event` fires. If waiting for the event takes too long (by default @@ -602,7 +83,7 @@ export class DatabaseManager extends DisposableObject { readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event; - private readonly _databaseItems: DatabaseItem[] = []; + private readonly _databaseItems: DatabaseItemImpl[] = []; private _currentDatabaseItem: DatabaseItem | undefined = undefined; constructor( @@ -646,8 +127,8 @@ export class DatabaseManager extends DisposableObject { * * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}. */ - public async addExistingDatabaseItem( - databaseItem: DatabaseItem, + private async addExistingDatabaseItem( + databaseItem: DatabaseItemImpl, progress: ProgressCallback, makeSelected: boolean, token: vscode.CancellationToken, @@ -681,7 +162,7 @@ export class DatabaseManager extends DisposableObject { private async createDatabaseItem( uri: vscode.Uri, displayName: string | undefined, - ): Promise { + ): Promise { const contents = await DatabaseResolver.resolveDatabaseContents(uri); // Ignore the source archive for QLTest databases by default. const isQLTestDatabase = extname(uri.fsPath) === ".testproj"; @@ -692,14 +173,7 @@ export class DatabaseManager extends DisposableObject { dateAdded: Date.now(), language: await this.getPrimaryLanguage(uri.fsPath), }; - const databaseItem = new DatabaseItemImpl( - uri, - contents, - fullOptions, - (event) => { - this._onDidChangeDatabaseItem.fire(event); - }, - ); + const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions); return databaseItem; } @@ -848,7 +322,7 @@ export class DatabaseManager extends DisposableObject { progress: ProgressCallback, token: vscode.CancellationToken, state: PersistedDatabaseItem, - ): Promise { + ): Promise { let displayName: string | undefined = undefined; let ignoreSourceArchive = false; let dateAdded = undefined; @@ -878,14 +352,7 @@ export class DatabaseManager extends DisposableObject { dateAdded, language, }; - const item = new DatabaseItemImpl( - dbBaseUri, - undefined, - fullOptions, - (event) => { - this._onDidChangeDatabaseItem.fire(event); - }, - ); + const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions); // Avoid persisting the database state after adding since that should happen only after // all databases have been added. @@ -926,7 +393,7 @@ export class DatabaseManager extends DisposableObject { database, ); try { - await databaseItem.refresh(); + await this.refreshDatabase(databaseItem); await this.registerDatabase(progress, token, databaseItem); if (currentDatabaseUri === database.uri) { await this.setCurrentDatabaseItem(databaseItem, true); @@ -968,8 +435,12 @@ export class DatabaseManager extends DisposableObject { item: DatabaseItem | undefined, skipRefresh = false, ): Promise { - if (!skipRefresh && item !== undefined) { - await item.refresh(); // Will throw on invalid database. + if ( + !skipRefresh && + item !== undefined && + item instanceof DatabaseItemImpl + ) { + await this.refreshDatabase(item); // Will throw on invalid database. } if (this._currentDatabaseItem !== item) { this._currentDatabaseItem = item; @@ -1018,7 +489,7 @@ export class DatabaseManager extends DisposableObject { private async addDatabaseItem( progress: ProgressCallback, token: vscode.CancellationToken, - item: DatabaseItem, + item: DatabaseItemImpl, updatePersistedState = true, ) { this._databaseItems.push(item); @@ -1135,6 +606,34 @@ export class DatabaseManager extends DisposableObject { await this.qs.registerDatabase(progress, token, dbItem); } + /** + * Resolves the contents of the database. + * + * @remarks + * The contents include the database directory, source archive, and metadata about the database. + * If the database is invalid, `databaseItem.error` is updated with the error object that describes why + * the database is invalid. This error is also thrown. + */ + private async refreshDatabase(databaseItem: DatabaseItemImpl) { + try { + try { + databaseItem.contents = await DatabaseResolver.resolveDatabaseContents( + databaseItem.databaseUri, + ); + databaseItem.error = undefined; + } catch (e) { + databaseItem.contents = undefined; + databaseItem.error = asError(e); + throw e; + } + } finally { + this._onDidChangeDatabaseItem.fire({ + kind: DatabaseEventKind.Refresh, + item: databaseItem, + }); + } + } + private updatePersistedCurrentDatabaseItem(): void { void this.ctx.workspaceState.update( CURRENT_DB, @@ -1164,15 +663,3 @@ export class DatabaseManager extends DisposableObject { return dbInfo.languages?.[0] || ""; } } - -/** - * Get the set of directories containing upgrades, given a list of - * scripts returned by the cli's upgrade resolution. - */ -export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] { - const parentDirs = scripts.map((dir) => dirname(dir)); - const uniqueParentDirs = new Set(parentDirs); - return Array.from(uniqueParentDirs).map((filePath) => - vscode.Uri.file(filePath), - ); -} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-options.ts b/extensions/ql-vscode/src/databases/local-databases/database-options.ts new file mode 100644 index 000000000..b8990e759 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-options.ts @@ -0,0 +1,12 @@ +export interface DatabaseOptions { + displayName?: string; + ignoreSourceArchive?: boolean; + dateAdded?: number | undefined; + language?: string; +} + +export interface FullDatabaseOptions extends DatabaseOptions { + ignoreSourceArchive: boolean; + dateAdded: number | undefined; + language: string | undefined; +} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts b/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts new file mode 100644 index 000000000..aa758c773 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts @@ -0,0 +1,144 @@ +import vscode from "vscode"; +import { pathExists } from "fs-extra"; +import { basename, join, resolve } from "path"; +import { + DatabaseContents, + DatabaseContentsWithDbScheme, + DatabaseKind, +} from "./database-contents"; +import { glob } from "glob"; +import { + showAndLogInformationMessage, + showAndLogWarningMessage, +} from "../../helpers"; +import { encodeArchiveBasePath } from "../../common/vscode/archive-filesystem-provider"; + +export class DatabaseResolver { + public static async resolveDatabaseContents( + uri: vscode.Uri, + ): Promise { + if (uri.scheme !== "file") { + throw new Error( + `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`, + ); + } + const databasePath = uri.fsPath; + if (!(await pathExists(databasePath))) { + throw new InvalidDatabaseError( + `Database '${databasePath}' does not exist.`, + ); + } + + const contents = await this.resolveDatabase(databasePath); + + if (contents === undefined) { + throw new InvalidDatabaseError( + `'${databasePath}' is not a valid database.`, + ); + } + + // Look for a single dbscheme file within the database. + // This should be found in the dataset directory, regardless of the form of database. + const dbPath = contents.datasetUri.fsPath; + const dbSchemeFiles = await getDbSchemeFiles(dbPath); + if (dbSchemeFiles.length === 0) { + throw new InvalidDatabaseError( + `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`, + ); + } else if (dbSchemeFiles.length > 1) { + throw new InvalidDatabaseError( + `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`, + ); + } else { + const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0])); + return { + ...contents, + dbSchemeUri, + }; + } + } + + public static async resolveDatabase( + databasePath: string, + ): Promise { + const name = basename(databasePath); + + // Look for dataset and source archive. + const datasetUri = await findDataset(databasePath); + const sourceArchiveUri = await findSourceArchive(databasePath); + + return { + kind: DatabaseKind.Database, + name, + datasetUri, + sourceArchiveUri, + }; + } +} + +/** + * An error thrown when we cannot find a valid database in a putative + * database directory. + */ +class InvalidDatabaseError extends Error {} + +async function findDataset(parentDirectory: string): Promise { + /* + * Look directly in the root + */ + let dbRelativePaths = await glob("db-*/", { + cwd: parentDirectory, + }); + + if (dbRelativePaths.length === 0) { + /* + * Check If they are in the old location + */ + dbRelativePaths = await glob("working/db-*/", { + cwd: parentDirectory, + }); + } + if (dbRelativePaths.length === 0) { + throw new InvalidDatabaseError( + `'${parentDirectory}' does not contain a dataset directory.`, + ); + } + + const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]); + if (dbRelativePaths.length > 1) { + void showAndLogWarningMessage( + `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`, + ); + } + + return vscode.Uri.file(dbAbsolutePath); +} + +/** Gets the relative paths of all `.dbscheme` files in the given directory. */ +async function getDbSchemeFiles(dbDirectory: string): Promise { + return await glob("*.dbscheme", { cwd: dbDirectory }); +} + +// exported for testing +export async function findSourceArchive( + databasePath: string, +): Promise { + const relativePaths = ["src", "output/src_archive"]; + + for (const relativePath of relativePaths) { + const basePath = join(databasePath, relativePath); + const zipPath = `${basePath}.zip`; + + // Prefer using a zip archive over a directory. + if (await pathExists(zipPath)) { + return encodeArchiveBasePath(zipPath); + } else if (await pathExists(basePath)) { + return vscode.Uri.file(basePath); + } + } + + void showAndLogInformationMessage( + `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`, + ); + return undefined; +} diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts new file mode 100644 index 000000000..fbca66f64 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/index.ts @@ -0,0 +1,11 @@ +export { + DatabaseContents, + DatabaseContentsWithDbScheme, + DatabaseKind, +} from "./database-contents"; +export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; +export { DatabaseItem } from "./database-item"; +export { DatabaseItemImpl } from "./database-item-impl"; +export { DatabaseManager } from "./database-manager"; +export { DatabaseResolver } from "./database-resolver"; +export { DatabaseOptions, FullDatabaseOptions } from "./database-options"; diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts index 16abf9f50..9172b2822 100644 --- a/extensions/ql-vscode/src/databases/ui/db-panel.ts +++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts @@ -1,4 +1,5 @@ import { + ProgressLocation, QuickPickItem, TreeView, TreeViewExpansionEvent, @@ -13,7 +14,10 @@ import { getOwnerFromGitHubUrl, isValidGitHubOwner, } from "../../common/github-url-identifier-helper"; -import { showAndLogErrorMessage } from "../../helpers"; +import { + showAndLogErrorMessage, + showAndLogInformationMessage, +} from "../../helpers"; import { DisposableObject } from "../../pure/disposable-object"; import { DbItem, @@ -32,6 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query"; import { getErrorMessage } from "../../pure/helpers-pure"; import { DatabasePanelCommands } from "../../common/commands"; import { App } from "../../common/app"; +import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client"; +import { QueryLanguage } from "../../common/query-language"; export interface RemoteDatabaseQuickPickItem extends QuickPickItem { remoteDatabaseKind: string; @@ -41,6 +47,10 @@ export interface AddListQuickPickItem extends QuickPickItem { databaseKind: DbListKind; } +export interface CodeSearchQuickPickItem extends QuickPickItem { + language: string; +} + export class DbPanel extends DisposableObject { private readonly dataProvider: DbTreeDataProvider; private readonly treeView: TreeView; @@ -93,6 +103,8 @@ export class DbPanel extends DisposableObject { this.renameItem.bind(this), "codeQLVariantAnalysisRepositories.removeItemContextMenu": this.removeItem.bind(this), + "codeQLVariantAnalysisRepositories.importFromCodeSearch": + this.importFromCodeSearch.bind(this), }; } @@ -171,7 +183,14 @@ export class DbPanel extends DisposableObject { return; } - await this.dbManager.addNewRemoteRepo(nwo, parentList); + const truncatedRepositories = await this.dbManager.addNewRemoteRepo( + nwo, + parentList, + ); + + if (parentList) { + this.reportAnyTruncatedRepos(truncatedRepositories, parentList); + } } private async addNewRemoteOwner(): Promise { @@ -323,6 +342,89 @@ export class DbPanel extends DisposableObject { await this.dbManager.removeDbItem(treeViewItem.dbItem); } + private async importFromCodeSearch( + treeViewItem: DbTreeViewItem, + ): Promise { + if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) { + throw new Error("Please select a valid list to add code search results."); + } + + const listName = treeViewItem.dbItem.listName; + + const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values( + QueryLanguage, + ).map((language) => ({ + label: language.toString(), + alwaysShow: true, + language: language.toString(), + })); + + const codeSearchLanguage = + await window.showQuickPick( + languageQuickPickItems, + { + title: "Select a language for your search", + placeHolder: "Select an option", + ignoreFocusOut: true, + }, + ); + if (!codeSearchLanguage) { + return; + } + + const codeSearchQuery = await window.showInputBox({ + title: "GitHub Code Search", + prompt: + "Use [GitHub's Code Search syntax](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax), including code qualifiers, regular expressions, and boolean operations, to search for repositories.", + placeHolder: "org:github", + }); + if (codeSearchQuery === undefined || codeSearchQuery === "") { + return; + } + + void window.withProgress( + { + location: ProgressLocation.Notification, + title: "Searching for repositories... This might take a while", + cancellable: true, + }, + async (progress, token) => { + progress.report({ increment: 10 }); + + const repositories = await getCodeSearchRepositories( + this.app.credentials, + `${codeSearchQuery} language:${codeSearchLanguage.language}`, + progress, + token, + ); + + token.onCancellationRequested(() => { + void showAndLogInformationMessage("Code search cancelled"); + return; + }); + + progress.report({ increment: 10, message: "Processing results..." }); + + const truncatedRepositories = + await this.dbManager.addNewRemoteReposToList(repositories, listName); + this.reportAnyTruncatedRepos(truncatedRepositories, listName); + }, + ); + } + + private reportAnyTruncatedRepos( + truncatedRepositories: string[], + listName: string, + ) { + if (truncatedRepositories.length > 0) { + void showAndLogErrorMessage( + `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join( + ", ", + )}`, + ); + } + } + private async onDidCollapseElement( event: TreeViewExpansionEvent, ): Promise { diff --git a/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts b/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts index 2755934f7..315b12c8b 100644 --- a/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts +++ b/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts @@ -4,7 +4,8 @@ export type DbTreeViewItemAction = | "canBeSelected" | "canBeRemoved" | "canBeRenamed" - | "canBeOpenedOnGitHub"; + | "canBeOpenedOnGitHub" + | "canImportCodeSearch"; export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] { const actions: DbTreeViewItemAction[] = []; @@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] { if (canBeOpenedOnGitHub(dbItem)) { actions.push("canBeOpenedOnGitHub"); } - + if (canImportCodeSearch(dbItem)) { + actions.push("canImportCodeSearch"); + } return actions; } @@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean { return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind); } +function canImportCodeSearch(dbItem: DbItem): boolean { + return DbItemKind.RemoteUserDefinedList === dbItem.kind; +} + export function getGitHubUrl(dbItem: DbItem): string | undefined { switch (dbItem.kind) { case DbItemKind.RemoteOwner: diff --git a/extensions/ql-vscode/src/debugger/debug-configuration.ts b/extensions/ql-vscode/src/debugger/debug-configuration.ts index c25e4fc6f..4d4a788a7 100644 --- a/extensions/ql-vscode/src/debugger/debug-configuration.ts +++ b/extensions/ql-vscode/src/debugger/debug-configuration.ts @@ -105,7 +105,7 @@ export class QLDebugConfigurationProvider validateQueryPath(qlConfiguration.query, quickEval); const quickEvalContext = quickEval - ? await getQuickEvalContext(undefined) + ? await getQuickEvalContext(undefined, false) : undefined; const resultConfiguration: QLResolvedDebugConfiguration = { diff --git a/extensions/ql-vscode/src/debugger/debug-session.ts b/extensions/ql-vscode/src/debugger/debug-session.ts index 9b9091092..02c183200 100644 --- a/extensions/ql-vscode/src/debugger/debug-session.ts +++ b/extensions/ql-vscode/src/debugger/debug-session.ts @@ -155,6 +155,7 @@ class RunningQuery extends DisposableObject { { queryPath: config.query, quickEvalPosition: quickEvalContext?.quickEvalPosition, + quickEvalCountOnly: quickEvalContext?.quickEvalCount, }, true, config.additionalPacks, diff --git a/extensions/ql-vscode/src/debugger/debugger-ui.ts b/extensions/ql-vscode/src/debugger/debugger-ui.ts index e6a115a43..46168e021 100644 --- a/extensions/ql-vscode/src/debugger/debugger-ui.ts +++ b/extensions/ql-vscode/src/debugger/debugger-ui.ts @@ -74,7 +74,7 @@ class QLDebugAdapterTracker public async quickEval(): Promise { const args: CodeQLProtocol.QuickEvalRequest["arguments"] = { - quickEvalContext: await getQuickEvalContext(undefined), + quickEvalContext: await getQuickEvalContext(undefined, false), }; await this.session.customRequest("codeql-quickeval", args); } diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index c6d4597e5..f96df9401 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -24,7 +24,7 @@ import { activate as archiveFilesystemProvider_activate, zipArchiveScheme, } from "./common/vscode/archive-filesystem-provider"; -import { CodeQLCliServer } from "./codeql-cli/cli"; +import { CliVersionConstraint, CodeQLCliServer } from "./codeql-cli/cli"; import { CliConfigListener, DistributionConfigListener, @@ -125,6 +125,7 @@ import { TestManager } from "./query-testing/test-manager"; import { TestRunner } from "./query-testing/test-runner"; import { TestManagerBase } from "./query-testing/test-manager-base"; import { NewQueryRunner, QueryRunner, QueryServerClient } from "./query-server"; +import { QueriesModule } from "./queries-panel/queries-module"; /** * extension.ts @@ -407,6 +408,28 @@ export async function activate( codeQlExtension.cliServer.addVersionChangedListener((ver) => { telemetryListener.cliVersion = ver; }); + + let unsupportedWarningShown = false; + codeQlExtension.cliServer.addVersionChangedListener((ver) => { + if (!ver) { + return; + } + + if (unsupportedWarningShown) { + return; + } + + if (CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION.compare(ver) < 0) { + return; + } + + void showAndLogWarningMessage( + `You are using an unsupported version of the CodeQL CLI (${ver}). ` + + `The minimum supported version is ${CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION}. ` + + `Please upgrade to a newer version of the CodeQL CLI.`, + ); + unsupportedWarningShown = true; + }); } return codeQlExtension; @@ -732,6 +755,8 @@ async function activateWithInstalledDistribution( ); ctx.subscriptions.push(databaseUI); + QueriesModule.initialize(app, cliServer); + void extLogger.log("Initializing evaluator log viewer."); const evalLogViewer = new EvalLogViewer(); ctx.subscriptions.push(evalLogViewer); diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts index e7e33f206..aa9c17985 100644 --- a/extensions/ql-vscode/src/helpers.ts +++ b/extensions/ql-vscode/src/helpers.ts @@ -10,14 +10,7 @@ import { glob } from "glob"; import { load } from "js-yaml"; import { join, basename, dirname } from "path"; import { dirSync } from "tmp-promise"; -import { - ExtensionContext, - Uri, - window as Window, - workspace, - env, - WorkspaceFolder, -} from "vscode"; +import { Uri, window as Window, workspace, env, WorkspaceFolder } from "vscode"; import { CodeQLCliServer, QlpacksInfo } from "./codeql-cli/cli"; import { UserCancellationException } from "./common/vscode/progress"; import { extLogger, OutputChannelLogger } from "./common"; @@ -98,7 +91,7 @@ export async function showAndLogErrorMessage( return internalShowAndLog( dropLinesExceptInitial(message), Window.showErrorMessage, - options, + { fullMessage: message, ...options }, ); } @@ -363,106 +356,6 @@ export async function prepareCodeTour( } } -/** - * Provides a utility method to invoke a function only if a minimum time interval has elapsed since - * the last invocation of that function. - */ -export class InvocationRateLimiter { - constructor( - extensionContext: ExtensionContext, - funcIdentifier: string, - func: () => Promise, - createDate: (dateString?: string) => Date = (s) => - s ? new Date(s) : new Date(), - ) { - this._createDate = createDate; - this._extensionContext = extensionContext; - this._func = func; - this._funcIdentifier = funcIdentifier; - } - - /** - * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation. - */ - public async invokeFunctionIfIntervalElapsed( - minSecondsSinceLastInvocation: number, - ): Promise> { - const updateCheckStartDate = this._createDate(); - const lastInvocationDate = this.getLastInvocationDate(); - if ( - minSecondsSinceLastInvocation && - lastInvocationDate && - lastInvocationDate <= updateCheckStartDate && - lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 > - updateCheckStartDate.getTime() - ) { - return createRateLimitedResult(); - } - const result = await this._func(); - await this.setLastInvocationDate(updateCheckStartDate); - return createInvokedResult(result); - } - - private getLastInvocationDate(): Date | undefined { - const maybeDateString: string | undefined = - this._extensionContext.globalState.get( - InvocationRateLimiter._invocationRateLimiterPrefix + - this._funcIdentifier, - ); - return maybeDateString ? this._createDate(maybeDateString) : undefined; - } - - private async setLastInvocationDate(date: Date): Promise { - return await this._extensionContext.globalState.update( - InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier, - date, - ); - } - - private readonly _createDate: (dateString?: string) => Date; - private readonly _extensionContext: ExtensionContext; - private readonly _func: () => Promise; - private readonly _funcIdentifier: string; - - private static readonly _invocationRateLimiterPrefix = - "invocationRateLimiter_lastInvocationDate_"; -} - -export enum InvocationRateLimiterResultKind { - Invoked, - RateLimited, -} - -/** - * The function was invoked and returned the value `result`. - */ -interface InvokedResult { - kind: InvocationRateLimiterResultKind.Invoked; - result: T; -} - -/** - * The function was not invoked as the minimum interval since the last invocation had not elapsed. - */ -interface RateLimitedResult { - kind: InvocationRateLimiterResultKind.RateLimited; -} - -type InvocationRateLimiterResult = InvokedResult | RateLimitedResult; - -function createInvokedResult(result: T): InvokedResult { - return { - kind: InvocationRateLimiterResultKind.Invoked, - result, - }; -} - -function createRateLimitedResult(): RateLimitedResult { - return { - kind: InvocationRateLimiterResultKind.RateLimited, - }; -} - export interface QlPacksForLanguage { /** The name of the pack containing the dbscheme. */ dbschemePack: string; @@ -584,77 +477,6 @@ export async function getPrimaryDbscheme( return dbscheme; } -/** - * A cached mapping from strings to value of type U. - */ -export class CachedOperation { - private readonly operation: (t: string, ...args: any[]) => Promise; - private readonly cached: Map; - private readonly lru: string[]; - private readonly inProgressCallbacks: Map< - string, - Array<[(u: U) => void, (reason?: any) => void]> - >; - - constructor( - operation: (t: string, ...args: any[]) => Promise, - private cacheSize = 100, - ) { - this.operation = operation; - this.lru = []; - this.inProgressCallbacks = new Map< - string, - Array<[(u: U) => void, (reason?: any) => void]> - >(); - this.cached = new Map(); - } - - async get(t: string, ...args: any[]): Promise { - // Try and retrieve from the cache - const fromCache = this.cached.get(t); - if (fromCache !== undefined) { - // Move to end of lru list - this.lru.push( - this.lru.splice( - this.lru.findIndex((v) => v === t), - 1, - )[0], - ); - return fromCache; - } - // Otherwise check if in progress - const inProgressCallback = this.inProgressCallbacks.get(t); - if (inProgressCallback !== undefined) { - // If so wait for it to resolve - return await new Promise((resolve, reject) => { - inProgressCallback.push([resolve, reject]); - }); - } - - // Otherwise compute the new value, but leave a callback to allow sharing work - const callbacks: Array<[(u: U) => void, (reason?: any) => void]> = []; - this.inProgressCallbacks.set(t, callbacks); - try { - const result = await this.operation(t, ...args); - callbacks.forEach((f) => f[0](result)); - this.inProgressCallbacks.delete(t); - if (this.lru.length > this.cacheSize) { - const toRemove = this.lru.shift()!; - this.cached.delete(toRemove); - } - this.lru.push(t); - this.cached.set(t, result); - return result; - } catch (e) { - // Rethrow error on all callbacks - callbacks.forEach((f) => f[1](e)); - throw e; - } finally { - this.inProgressCallbacks.delete(t); - } - } -} - /** * The following functions al heuristically determine metadata about databases. */ diff --git a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts index 3325c9df0..76e6eed06 100644 --- a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts +++ b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts @@ -2,7 +2,7 @@ import { Uri, window } from "vscode"; import { withProgress } from "../../common/vscode/progress"; import { AstViewer } from "./ast-viewer"; import { AstCfgCommands } from "../../common/commands"; -import { LocalQueries } from "../../local-queries"; +import { LocalQueries, QuickEvalType } from "../../local-queries"; import { TemplatePrintAstProvider, TemplatePrintCfgProvider, @@ -42,12 +42,17 @@ export function getAstCfgCommands({ const viewCfg = async () => withProgress( async (progress, token) => { - const res = await cfgTemplateProvider.provideCfgUri( - window.activeTextEditor?.document, - ); + const editor = window.activeTextEditor; + const res = !editor + ? undefined + : await cfgTemplateProvider.provideCfgUri( + editor.document, + editor.selection.active.line + 1, + editor.selection.active.character + 1, + ); if (res) { await localQueries.compileAndRunQuery( - false, + QuickEvalType.None, res[0], progress, token, diff --git a/extensions/ql-vscode/src/language-support/contextual/cached-operation.ts b/extensions/ql-vscode/src/language-support/contextual/cached-operation.ts new file mode 100644 index 000000000..c51970de1 --- /dev/null +++ b/extensions/ql-vscode/src/language-support/contextual/cached-operation.ts @@ -0,0 +1,70 @@ +/** + * A cached mapping from strings to value of type U. + */ +export class CachedOperation { + private readonly operation: (t: string, ...args: any[]) => Promise; + private readonly cached: Map; + private readonly lru: string[]; + private readonly inProgressCallbacks: Map< + string, + Array<[(u: U) => void, (reason?: any) => void]> + >; + + constructor( + operation: (t: string, ...args: any[]) => Promise, + private cacheSize = 100, + ) { + this.operation = operation; + this.lru = []; + this.inProgressCallbacks = new Map< + string, + Array<[(u: U) => void, (reason?: any) => void]> + >(); + this.cached = new Map(); + } + + async get(t: string, ...args: any[]): Promise { + // Try and retrieve from the cache + const fromCache = this.cached.get(t); + if (fromCache !== undefined) { + // Move to end of lru list + this.lru.push( + this.lru.splice( + this.lru.findIndex((v) => v === t), + 1, + )[0], + ); + return fromCache; + } + // Otherwise check if in progress + const inProgressCallback = this.inProgressCallbacks.get(t); + if (inProgressCallback !== undefined) { + // If so wait for it to resolve + return await new Promise((resolve, reject) => { + inProgressCallback.push([resolve, reject]); + }); + } + + // Otherwise compute the new value, but leave a callback to allow sharing work + const callbacks: Array<[(u: U) => void, (reason?: any) => void]> = []; + this.inProgressCallbacks.set(t, callbacks); + try { + const result = await this.operation(t, ...args); + callbacks.forEach((f) => f[0](result)); + this.inProgressCallbacks.delete(t); + if (this.lru.length > this.cacheSize) { + const toRemove = this.lru.shift()!; + this.cached.delete(toRemove); + } + this.lru.push(t); + this.cached.set(t, result); + return result; + } catch (e) { + // Rethrow error on all callbacks + callbacks.forEach((f) => f[1](e)); + throw e; + } finally { + this.inProgressCallbacks.delete(t); + } + } +} diff --git a/extensions/ql-vscode/src/language-support/contextual/location-finder.ts b/extensions/ql-vscode/src/language-support/contextual/location-finder.ts index ef998a10a..980327475 100644 --- a/extensions/ql-vscode/src/language-support/contextual/location-finder.ts +++ b/extensions/ql-vscode/src/language-support/contextual/location-finder.ts @@ -24,7 +24,9 @@ import { QueryResultType } from "../../pure/new-messages"; import { fileRangeFromURI } from "./file-range-from-uri"; export const SELECT_QUERY_NAME = "#select"; -export const TEMPLATE_NAME = "selectedSourceFile"; +export const SELECTED_SOURCE_FILE = "selectedSourceFile"; +export const SELECTED_SOURCE_LINE = "selectedSourceLine"; +export const SELECTED_SOURCE_COLUMN = "selectedSourceColumn"; export interface FullLocationLink extends LocationLink { originUri: Uri; @@ -124,7 +126,7 @@ async function getLinksFromResults( function createTemplates(path: string): Record { return { - [TEMPLATE_NAME]: path, + [SELECTED_SOURCE_FILE]: path, }; } diff --git a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts index 26ba9d72f..7943f4513 100644 --- a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts +++ b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts @@ -17,13 +17,15 @@ import { } from "../../common/vscode/archive-filesystem-provider"; import { CodeQLCliServer } from "../../codeql-cli/cli"; import { DatabaseManager } from "../../databases/local-databases"; -import { CachedOperation } from "../../helpers"; +import { CachedOperation } from "./cached-operation"; import { ProgressCallback, withProgress } from "../../common/vscode/progress"; import { KeyType } from "./key-type"; import { FullLocationLink, getLocationsForUriString, - TEMPLATE_NAME, + SELECTED_SOURCE_FILE, + SELECTED_SOURCE_LINE, + SELECTED_SOURCE_COLUMN, } from "./location-finder"; import { qlpackOfDatabase, @@ -253,7 +255,7 @@ export class TemplatePrintAstProvider { const query = queries[0]; const templates: Record = { - [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive, + [SELECTED_SOURCE_FILE]: zippedArchive.pathWithinSourceArchive, }; const results = await runContextualQuery( @@ -284,15 +286,17 @@ export class TemplatePrintCfgProvider { } async provideCfgUri( - document?: TextDocument, + document: TextDocument, + line: number, + character: number, ): Promise<[Uri, Record] | undefined> { - if (!document) { - return; - } - return this.shouldUseCache() - ? await this.cache.get(document.uri.toString()) - : await this.getCfgUri(document.uri.toString()); + ? await this.cache.get( + `${document.uri.toString()}#${line}:${character}`, + line, + character, + ) + : await this.getCfgUri(document.uri.toString(), line, character); } private shouldUseCache() { @@ -301,6 +305,8 @@ export class TemplatePrintCfgProvider { private async getCfgUri( uriString: string, + line: number, + character: number, ): Promise<[Uri, Record]> { const uri = Uri.parse(uriString, true); if (uri.scheme !== zipArchiveScheme) { @@ -342,7 +348,9 @@ export class TemplatePrintCfgProvider { const queryUri = Uri.file(queries[0]); const templates: Record = { - [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive, + [SELECTED_SOURCE_FILE]: zippedArchive.pathWithinSourceArchive, + [SELECTED_SOURCE_LINE]: line.toString(), + [SELECTED_SOURCE_COLUMN]: character.toString(), }; return [queryUri, templates]; diff --git a/extensions/ql-vscode/src/local-queries/local-queries.ts b/extensions/ql-vscode/src/local-queries/local-queries.ts index b7250cbb4..fa69f256f 100644 --- a/extensions/ql-vscode/src/local-queries/local-queries.ts +++ b/extensions/ql-vscode/src/local-queries/local-queries.ts @@ -47,7 +47,7 @@ import { App } from "../common/app"; import { DisposableObject } from "../pure/disposable-object"; import { SkeletonQueryWizard } from "../skeleton-query-wizard"; import { LocalQueryRun } from "./local-query-run"; -import { createMultiSelectionCommand } from "../common/selection-commands"; +import { createMultiSelectionCommand } from "../common/vscode/selection-commands"; interface DatabaseQuickPickItem extends QuickPickItem { databaseItem: DatabaseItem; @@ -72,6 +72,12 @@ async function promptToSaveQueryIfNeeded(query: SelectedQuery): Promise { } } +export enum QuickEvalType { + None, + QuickEval, + QuickEvalCount, +} + export class LocalQueries extends DisposableObject { public constructor( private readonly app: App, @@ -115,7 +121,13 @@ export class LocalQueries extends DisposableObject { private async runQuery(uri: Uri | undefined): Promise { await withProgress( async (progress, token) => { - await this.compileAndRunQuery(false, uri, progress, token, undefined); + await this.compileAndRunQuery( + QuickEvalType.None, + uri, + progress, + token, + undefined, + ); }, { title: "Running query", @@ -185,7 +197,7 @@ export class LocalQueries extends DisposableObject { await Promise.all( queryUris.map(async (uri) => this.compileAndRunQuery( - false, + QuickEvalType.None, uri, wrappedProgress, token, @@ -204,7 +216,13 @@ export class LocalQueries extends DisposableObject { private async quickEval(uri: Uri): Promise { await withProgress( async (progress, token) => { - await this.compileAndRunQuery(true, uri, progress, token, undefined); + await this.compileAndRunQuery( + QuickEvalType.QuickEval, + uri, + progress, + token, + undefined, + ); }, { title: "Running query", @@ -217,7 +235,7 @@ export class LocalQueries extends DisposableObject { await withProgress( async (progress, token) => await this.compileAndRunQuery( - true, + QuickEvalType.QuickEval, uri, progress, token, @@ -331,7 +349,7 @@ export class LocalQueries extends DisposableObject { } public async compileAndRunQuery( - quickEval: boolean, + quickEval: QuickEvalType, queryUri: Uri | undefined, progress: ProgressCallback, token: CancellationToken, @@ -352,7 +370,7 @@ export class LocalQueries extends DisposableObject { /** Used by tests */ public async compileAndRunQueryInternal( - quickEval: boolean, + quickEval: QuickEvalType, queryUri: Uri | undefined, progress: ProgressCallback, token: CancellationToken, @@ -364,15 +382,20 @@ export class LocalQueries extends DisposableObject { if (queryUri !== undefined) { // The query URI is provided by the command, most likely because the command was run from an // editor context menu. Use the provided URI, but make sure it's a valid query. - queryPath = validateQueryUri(queryUri, quickEval); + queryPath = validateQueryUri(queryUri, quickEval !== QuickEvalType.None); } else { // Use the currently selected query. - queryPath = await this.getCurrentQuery(quickEval); + queryPath = await this.getCurrentQuery(quickEval !== QuickEvalType.None); } const selectedQuery: SelectedQuery = { queryPath, - quickEval: quickEval ? await getQuickEvalContext(range) : undefined, + quickEval: quickEval + ? await getQuickEvalContext( + range, + quickEval === QuickEvalType.QuickEvalCount, + ) + : undefined, }; // If no databaseItem is specified, use the database currently selected in the Databases UI @@ -392,6 +415,7 @@ export class LocalQueries extends DisposableObject { { queryPath: selectedQuery.queryPath, quickEvalPosition: selectedQuery.quickEval?.quickEvalPosition, + quickEvalCountOnly: selectedQuery.quickEval?.quickEvalCount, }, true, additionalPacks, @@ -481,7 +505,7 @@ export class LocalQueries extends DisposableObject { for (const item of quickpick) { try { await this.compileAndRunQuery( - false, + QuickEvalType.None, uri, progress, token, diff --git a/extensions/ql-vscode/src/pure/bqrs-cli-types.ts b/extensions/ql-vscode/src/pure/bqrs-cli-types.ts index 1a8be81be..ae8c35092 100644 --- a/extensions/ql-vscode/src/pure/bqrs-cli-types.ts +++ b/extensions/ql-vscode/src/pure/bqrs-cli-types.ts @@ -115,7 +115,7 @@ export type BqrsKind = | "Entity"; interface BqrsColumn { - name: string; + name?: string; kind: BqrsKind; } export interface DecodedBqrsChunk { diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index c04644b73..a71591745 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -544,6 +544,12 @@ export interface GenerateExternalApiMessage { t: "generateExternalApi"; } +export interface GenerateExternalApiFromLlmMessage { + t: "generateExternalApiFromLlm"; + externalApiUsages: ExternalApiUsage[]; + modeledMethods: Record; +} + export type ToDataExtensionsEditorMessage = | SetExtensionPackStateMessage | SetExternalApiUsagesMessage @@ -556,4 +562,5 @@ export type FromDataExtensionsEditorMessage = | OpenExtensionPackMessage | JumpToUsageMessage | SaveModeledMethods - | GenerateExternalApiMessage; + | GenerateExternalApiMessage + | GenerateExternalApiFromLlmMessage; diff --git a/extensions/ql-vscode/src/pure/messages-shared.ts b/extensions/ql-vscode/src/pure/messages-shared.ts index 23b5a6871..a07d81f82 100644 --- a/extensions/ql-vscode/src/pure/messages-shared.ts +++ b/extensions/ql-vscode/src/pure/messages-shared.ts @@ -68,6 +68,14 @@ export interface CompilationTarget { */ export interface QuickEvalOptions { quickEvalPos?: Position; + /** + * Whether to only count the number of results. + * + * This is only supported by the new query server + * but it isn't worth having a separate type and + * it is fine to have an ignored optional field. + */ + countOnly?: boolean; } /** diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts new file mode 100644 index 000000000..aede56a8d --- /dev/null +++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts @@ -0,0 +1,40 @@ +import { CodeQLCliServer } from "../codeql-cli/cli"; +import { extLogger } from "../common"; +import { App, AppMode } from "../common/app"; +import { isCanary, showQueriesPanel } from "../config"; +import { DisposableObject } from "../pure/disposable-object"; +import { QueriesPanel } from "./queries-panel"; +import { QueryDiscovery } from "./query-discovery"; + +export class QueriesModule extends DisposableObject { + private constructor(readonly app: App) { + super(); + } + + private initialize(app: App, cliServer: CodeQLCliServer): void { + if (app.mode === AppMode.Production || !isCanary() || !showQueriesPanel()) { + // Currently, we only want to expose the new panel when we are in development and canary mode + // and the developer has enabled the "Show queries panel" flag. + return; + } + void extLogger.log("Initializing queries panel."); + + const queryDiscovery = new QueryDiscovery(app, cliServer); + this.push(queryDiscovery); + void queryDiscovery.refresh(); + + const queriesPanel = new QueriesPanel(queryDiscovery); + this.push(queriesPanel); + } + + public static initialize( + app: App, + cliServer: CodeQLCliServer, + ): QueriesModule { + const queriesModule = new QueriesModule(app); + app.subscriptions.push(queriesModule); + + queriesModule.initialize(app, cliServer); + return queriesModule; + } +} diff --git a/extensions/ql-vscode/src/queries-panel/queries-panel.ts b/extensions/ql-vscode/src/queries-panel/queries-panel.ts new file mode 100644 index 000000000..5988509f8 --- /dev/null +++ b/extensions/ql-vscode/src/queries-panel/queries-panel.ts @@ -0,0 +1,17 @@ +import * as vscode from "vscode"; +import { DisposableObject } from "../pure/disposable-object"; +import { QueryTreeDataProvider } from "./query-tree-data-provider"; +import { QueryDiscovery } from "./query-discovery"; + +export class QueriesPanel extends DisposableObject { + public constructor(queryDiscovery: QueryDiscovery) { + super(); + + const dataProvider = new QueryTreeDataProvider(queryDiscovery); + + const treeView = vscode.window.createTreeView("codeQLQueries", { + treeDataProvider: dataProvider, + }); + this.push(treeView); + } +} diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts new file mode 100644 index 000000000..f7b25ea09 --- /dev/null +++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts @@ -0,0 +1,140 @@ +import { dirname, basename, normalize, relative } from "path"; +import { Discovery } from "../common/discovery"; +import { CodeQLCliServer } from "../codeql-cli/cli"; +import { Event, RelativePattern, Uri, WorkspaceFolder } from "vscode"; +import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher"; +import { App } from "../common/app"; +import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes"; +import { getOnDiskWorkspaceFoldersObjects } from "../helpers"; +import { AppEventEmitter } from "../common/events"; +import { QueryDiscoverer } from "./query-tree-data-provider"; +import { extLogger } from "../common"; + +/** + * The results of discovering queries. + */ +export interface QueryDiscoveryResults { + /** + * A tree of directories and query files. + * May have multiple roots because of multiple workspaces. + */ + queries: FileTreeDirectory[]; + + /** + * File system paths to watch. If any ql file changes in these directories + * or any subdirectories, then this could signify a change in queries. + */ + watchPaths: Uri[]; +} + +/** + * Discovers all query files contained in the QL packs in a given workspace folder. + */ +export class QueryDiscovery + extends Discovery + implements QueryDiscoverer +{ + private results: QueryDiscoveryResults | undefined; + + private readonly onDidChangeQueriesEmitter: AppEventEmitter; + private readonly watcher: MultiFileSystemWatcher = this.push( + new MultiFileSystemWatcher(), + ); + + constructor(app: App, private readonly cliServer: CodeQLCliServer) { + super("Query Discovery", extLogger); + + this.onDidChangeQueriesEmitter = this.push(app.createEventEmitter()); + this.push(app.onDidChangeWorkspaceFolders(this.refresh.bind(this))); + this.push(this.watcher.onDidChange(this.refresh.bind(this))); + } + + public get queries(): FileTreeDirectory[] | undefined { + return this.results?.queries; + } + + /** + * Event to be fired when the set of discovered queries may have changed. + */ + public get onDidChangeQueries(): Event { + return this.onDidChangeQueriesEmitter.event; + } + + protected async discover(): Promise { + const workspaceFolders = getOnDiskWorkspaceFoldersObjects(); + if (workspaceFolders.length === 0) { + return { + queries: [], + watchPaths: [], + }; + } + + const queries = await this.discoverQueries(workspaceFolders); + + return { + queries, + watchPaths: workspaceFolders.map((f) => f.uri), + }; + } + + protected update(results: QueryDiscoveryResults): void { + this.results = results; + + this.watcher.clear(); + for (const watchPath of results.watchPaths) { + // Watch for changes to any `.ql` file + this.watcher.addWatch(new RelativePattern(watchPath, "**/*.{ql}")); + // need to explicitly watch for changes to directories themselves. + this.watcher.addWatch(new RelativePattern(watchPath, "**/")); + } + this.onDidChangeQueriesEmitter.fire(); + } + + /** + * Discover all queries in the specified directory and its subdirectories. + * @returns A `QueryDirectory` object describing the contents of the directory, or `undefined` if + * no queries were found. + */ + private async discoverQueries( + workspaceFolders: readonly WorkspaceFolder[], + ): Promise { + const rootDirectories = []; + for (const workspaceFolder of workspaceFolders) { + const root = await this.discoverQueriesInWorkspace(workspaceFolder); + if (root !== undefined) { + rootDirectories.push(root); + } + } + return rootDirectories; + } + + private async discoverQueriesInWorkspace( + workspaceFolder: WorkspaceFolder, + ): Promise { + const fullPath = workspaceFolder.uri.fsPath; + const name = workspaceFolder.name; + + // We don't want to log each invocation of resolveQueries, since it clutters up the log. + const silent = true; + const resolvedQueries = await this.cliServer.resolveQueries( + fullPath, + silent, + ); + if (resolvedQueries.length === 0) { + return undefined; + } + + const rootDirectory = new FileTreeDirectory(fullPath, name); + for (const queryPath of resolvedQueries) { + const relativePath = normalize(relative(fullPath, queryPath)); + const dirName = dirname(relativePath); + const parentDirectory = rootDirectory.createDirectory(dirName); + parentDirectory.addChild( + new FileTreeLeaf(queryPath, basename(queryPath)), + ); + } + + rootDirectory.finish(); + return rootDirectory; + } +} diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts new file mode 100644 index 000000000..0810ded8a --- /dev/null +++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts @@ -0,0 +1,74 @@ +import { Event, EventEmitter, TreeDataProvider, TreeItem } from "vscode"; +import { QueryTreeViewItem } from "./query-tree-view-item"; +import { DisposableObject } from "../pure/disposable-object"; +import { FileTreeNode } from "../common/file-tree-nodes"; + +export interface QueryDiscoverer { + readonly queries: FileTreeNode[] | undefined; + readonly onDidChangeQueries: Event; +} + +export class QueryTreeDataProvider + extends DisposableObject + implements TreeDataProvider +{ + private queryTreeItems: QueryTreeViewItem[]; + + private readonly onDidChangeTreeDataEmitter = this.push( + new EventEmitter(), + ); + + public constructor(private readonly queryDiscoverer: QueryDiscoverer) { + super(); + + queryDiscoverer.onDidChangeQueries(() => { + this.queryTreeItems = this.createTree(); + this.onDidChangeTreeDataEmitter.fire(); + }); + + this.queryTreeItems = this.createTree(); + } + + public get onDidChangeTreeData(): Event { + return this.onDidChangeTreeDataEmitter.event; + } + + private createTree(): QueryTreeViewItem[] { + return (this.queryDiscoverer.queries || []).map( + this.convertFileTreeNode.bind(this), + ); + } + + private convertFileTreeNode( + fileTreeDirectory: FileTreeNode, + ): QueryTreeViewItem { + return new QueryTreeViewItem( + fileTreeDirectory.name, + fileTreeDirectory.path, + fileTreeDirectory.children.map(this.convertFileTreeNode.bind(this)), + ); + } + + /** + * Returns the UI presentation of the element that gets displayed in the view. + * @param item The item to represent. + * @returns The UI presentation of the item. + */ + public getTreeItem(item: QueryTreeViewItem): TreeItem { + return item; + } + + /** + * Called when expanding an item (including the root item). + * @param item The item to expand. + * @returns The children of the item. + */ + public getChildren(item?: QueryTreeViewItem): QueryTreeViewItem[] { + if (!item) { + // We're at the root. + return this.queryTreeItems; + } else { + return item.children; + } + } +} diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts new file mode 100644 index 000000000..17caea32d --- /dev/null +++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts @@ -0,0 +1,22 @@ +import * as vscode from "vscode"; + +export class QueryTreeViewItem extends vscode.TreeItem { + constructor( + name: string, + path: string, + public readonly children: QueryTreeViewItem[], + ) { + super(name); + this.tooltip = path; + this.collapsibleState = this.children.length + ? vscode.TreeItemCollapsibleState.Collapsed + : vscode.TreeItemCollapsibleState.None; + if (this.children.length === 0) { + this.command = { + title: "Open", + command: "vscode.open", + arguments: [vscode.Uri.file(path)], + }; + } + } +} diff --git a/extensions/ql-vscode/src/query-history/query-history-manager.ts b/extensions/ql-vscode/src/query-history/query-history-manager.ts index ac24cd0bf..2b42c3ddf 100644 --- a/extensions/ql-vscode/src/query-history/query-history-manager.ts +++ b/extensions/ql-vscode/src/query-history/query-history-manager.ts @@ -59,7 +59,7 @@ import { tryOpenExternalFile } from "../common/vscode/external-files"; import { createMultiSelectionCommand, createSingleSelectionCommand, -} from "../common/selection-commands"; +} from "../common/vscode/selection-commands"; /** * query-history-manager.ts diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts index 0cab84e9c..77b3461ca 100644 --- a/extensions/ql-vscode/src/query-results.ts +++ b/extensions/ql-vscode/src/query-results.ts @@ -139,6 +139,7 @@ export async function interpretResultsSarif( metadata: QueryMetadata | undefined, resultsPaths: ResultsPaths, sourceInfo?: cli.SourceInfo, + args?: string[], ): Promise { const { resultsPath, interpretedResultsPath } = resultsPaths; let res; @@ -150,6 +151,7 @@ export async function interpretResultsSarif( resultsPath, interpretedResultsPath, sourceInfo, + args, ); } return { ...res, t: "SarifInterpretationData" }; diff --git a/extensions/ql-vscode/src/query-server/query-runner.ts b/extensions/ql-vscode/src/query-server/query-runner.ts index 95d7eafef..03f886ffd 100644 --- a/extensions/ql-vscode/src/query-server/query-runner.ts +++ b/extensions/ql-vscode/src/query-server/query-runner.ts @@ -16,6 +16,10 @@ export interface CoreQueryTarget { * `query`. */ quickEvalPosition?: Position; + /** + * If this is quick eval, whether to only count the number of results. + */ + quickEvalCountOnly?: boolean; } export interface CoreQueryResults { diff --git a/extensions/ql-vscode/src/query-server/run-queries.ts b/extensions/ql-vscode/src/query-server/run-queries.ts index 9f9e441c6..66674bab9 100644 --- a/extensions/ql-vscode/src/query-server/run-queries.ts +++ b/extensions/ql-vscode/src/query-server/run-queries.ts @@ -36,7 +36,10 @@ export async function compileAndRunQueryAgainstDatabaseCore( const target = query.quickEvalPosition !== undefined ? { - quickEval: { quickEvalPos: query.quickEvalPosition }, + quickEval: { + quickEvalPos: query.quickEvalPosition, + countOnly: query.quickEvalCountOnly, + }, } : { query: {} }; diff --git a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts index 891d2c965..5a9a4088f 100644 --- a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts +++ b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts @@ -1,4 +1,4 @@ -import { dirname, basename, join, normalize, relative, extname } from "path"; +import { dirname, basename, normalize, relative, extname } from "path"; import { Discovery } from "../common/discovery"; import { EventEmitter, @@ -6,112 +6,12 @@ import { Uri, RelativePattern, WorkspaceFolder, - env, } from "vscode"; import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher"; import { CodeQLCliServer } from "../codeql-cli/cli"; import { pathExists } from "fs-extra"; - -/** - * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`. - */ -export abstract class QLTestNode { - constructor(private _path: string, private _name: string) {} - - public get path(): string { - return this._path; - } - - public get name(): string { - return this._name; - } - - public abstract get children(): readonly QLTestNode[]; - - public abstract finish(): void; -} - -/** - * A directory containing one or more QL tests or other test directories. - */ -export class QLTestDirectory extends QLTestNode { - constructor( - _path: string, - _name: string, - private _children: QLTestNode[] = [], - ) { - super(_path, _name); - } - - public get children(): readonly QLTestNode[] { - return this._children; - } - - public addChild(child: QLTestNode): void { - this._children.push(child); - } - - public createDirectory(relativePath: string): QLTestDirectory { - const dirName = dirname(relativePath); - if (dirName === ".") { - return this.createChildDirectory(relativePath); - } else { - const parent = this.createDirectory(dirName); - return parent.createDirectory(basename(relativePath)); - } - } - - public finish(): void { - // remove empty directories - this._children.filter( - (child) => child instanceof QLTestFile || child.children.length > 0, - ); - this._children.sort((a, b) => a.name.localeCompare(b.name, env.language)); - this._children.forEach((child, i) => { - child.finish(); - if ( - child.children?.length === 1 && - child.children[0] instanceof QLTestDirectory - ) { - // collapse children - const replacement = new QLTestDirectory( - child.children[0].path, - `${child.name} / ${child.children[0].name}`, - Array.from(child.children[0].children), - ); - this._children[i] = replacement; - } - }); - } - - private createChildDirectory(name: string): QLTestDirectory { - const existingChild = this._children.find((child) => child.name === name); - if (existingChild !== undefined) { - return existingChild as QLTestDirectory; - } else { - const newChild = new QLTestDirectory(join(this.path, name), name); - this.addChild(newChild); - return newChild; - } - } -} - -/** - * A single QL test. This will be either a `.ql` file or a `.qlref` file. - */ -export class QLTestFile extends QLTestNode { - constructor(_path: string, _name: string) { - super(_path, _name); - } - - public get children(): readonly QLTestNode[] { - return []; - } - - public finish(): void { - /**/ - } -} +import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes"; +import { extLogger } from "../common"; /** * The results of discovering QL tests. @@ -120,7 +20,7 @@ interface QLTestDiscoveryResults { /** * A directory that contains one or more QL Tests, or other QLTestDirectories. */ - testDirectory: QLTestDirectory | undefined; + testDirectory: FileTreeDirectory | undefined; /** * The file system path to a directory to watch. If any ql or qlref file changes in @@ -137,13 +37,13 @@ export class QLTestDiscovery extends Discovery { private readonly watcher: MultiFileSystemWatcher = this.push( new MultiFileSystemWatcher(), ); - private _testDirectory: QLTestDirectory | undefined; + private _testDirectory: FileTreeDirectory | undefined; constructor( private readonly workspaceFolder: WorkspaceFolder, private readonly cliServer: CodeQLCliServer, ) { - super("QL Test Discovery"); + super("QL Test Discovery", extLogger); this.push(this.watcher.onDidChange(this.handleDidChange, this)); } @@ -159,13 +59,13 @@ export class QLTestDiscovery extends Discovery { * The root directory. There is at least one test in this directory, or * in a subdirectory of this. */ - public get testDirectory(): QLTestDirectory | undefined { + public get testDirectory(): FileTreeDirectory | undefined { return this._testDirectory; } private handleDidChange(uri: Uri): void { if (!QLTestDiscovery.ignoreTestPath(uri.fsPath)) { - this.refresh(); + void this.refresh(); } } protected async discover(): Promise { @@ -194,10 +94,10 @@ export class QLTestDiscovery extends Discovery { * @returns A `QLTestDirectory` object describing the contents of the directory, or `undefined` if * no tests were found. */ - private async discoverTests(): Promise { + private async discoverTests(): Promise { const fullPath = this.workspaceFolder.uri.fsPath; const name = this.workspaceFolder.name; - const rootDirectory = new QLTestDirectory(fullPath, name); + const rootDirectory = new FileTreeDirectory(fullPath, name); // Don't try discovery on workspace folders that don't exist on the filesystem if (await pathExists(fullPath)) { @@ -208,7 +108,9 @@ export class QLTestDiscovery extends Discovery { const relativePath = normalize(relative(fullPath, testPath)); const dirName = dirname(relativePath); const parentDirectory = rootDirectory.createDirectory(dirName); - parentDirectory.addChild(new QLTestFile(testPath, basename(testPath))); + parentDirectory.addChild( + new FileTreeLeaf(testPath, basename(testPath)), + ); } rootDirectory.finish(); diff --git a/extensions/ql-vscode/src/query-testing/test-adapter.ts b/extensions/ql-vscode/src/query-testing/test-adapter.ts index dae8f9780..85b30a47c 100644 --- a/extensions/ql-vscode/src/query-testing/test-adapter.ts +++ b/extensions/ql-vscode/src/query-testing/test-adapter.ts @@ -13,17 +13,17 @@ import { TestHub, } from "vscode-test-adapter-api"; import { TestAdapterRegistrar } from "vscode-test-adapter-util"; -import { - QLTestFile, - QLTestNode, - QLTestDirectory, - QLTestDiscovery, -} from "./qltest-discovery"; +import { QLTestDiscovery } from "./qltest-discovery"; import { Event, EventEmitter, CancellationTokenSource } from "vscode"; import { DisposableObject } from "../pure/disposable-object"; import { CodeQLCliServer, TestCompleted } from "../codeql-cli/cli"; import { testLogger } from "../common"; import { TestRunner } from "./test-runner"; +import { + FileTreeDirectory, + FileTreeLeaf, + FileTreeNode, +} from "../common/file-tree-nodes"; /** * Get the full path of the `.expected` file for the specified QL test. @@ -115,7 +115,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter { this.qlTestDiscovery = this.push( new QLTestDiscovery(workspaceFolder, cliServer), ); - this.qlTestDiscovery.refresh(); + void this.qlTestDiscovery.refresh(); this.push(this.qlTestDiscovery.onDidChangeTests(this.discoverTests, this)); } @@ -135,7 +135,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter { } private static createTestOrSuiteInfos( - testNodes: readonly QLTestNode[], + testNodes: readonly FileTreeNode[], ): Array { return testNodes.map((childNode) => { return QLTestAdapter.createTestOrSuiteInfo(childNode); @@ -143,18 +143,18 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter { } private static createTestOrSuiteInfo( - testNode: QLTestNode, + testNode: FileTreeNode, ): TestSuiteInfo | TestInfo { - if (testNode instanceof QLTestFile) { + if (testNode instanceof FileTreeLeaf) { return QLTestAdapter.createTestInfo(testNode); - } else if (testNode instanceof QLTestDirectory) { + } else if (testNode instanceof FileTreeDirectory) { return QLTestAdapter.createTestSuiteInfo(testNode, testNode.name); } else { throw new Error("Unexpected test type."); } } - private static createTestInfo(testFile: QLTestFile): TestInfo { + private static createTestInfo(testFile: FileTreeLeaf): TestInfo { return { type: "test", id: testFile.path, @@ -165,7 +165,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter { } private static createTestSuiteInfo( - testDirectory: QLTestDirectory, + testDirectory: FileTreeDirectory, label: string, ): TestSuiteInfo { return { diff --git a/extensions/ql-vscode/src/query-testing/test-manager.ts b/extensions/ql-vscode/src/query-testing/test-manager.ts index fc5789b8e..073cc7170 100644 --- a/extensions/ql-vscode/src/query-testing/test-manager.ts +++ b/extensions/ql-vscode/src/query-testing/test-manager.ts @@ -16,12 +16,7 @@ import { workspace, } from "vscode"; import { DisposableObject } from "../pure/disposable-object"; -import { - QLTestDirectory, - QLTestDiscovery, - QLTestFile, - QLTestNode, -} from "./qltest-discovery"; +import { QLTestDiscovery } from "./qltest-discovery"; import { CodeQLCliServer } from "../codeql-cli/cli"; import { getErrorMessage } from "../pure/helpers-pure"; import { BaseLogger, LogOptions } from "../common"; @@ -29,6 +24,11 @@ import { TestRunner } from "./test-runner"; import { TestManagerBase } from "./test-manager-base"; import { App } from "../common/app"; import { isWorkspaceFolderOnDisk } from "../helpers"; +import { + FileTreeDirectory, + FileTreeLeaf, + FileTreeNode, +} from "../common/file-tree-nodes"; /** * Returns the complete text content of the specified file. If there is an error reading the file, @@ -92,7 +92,7 @@ class WorkspaceFolderHandler extends DisposableObject { this.push( this.testDiscovery.onDidChangeTests(this.handleDidChangeTests, this), ); - this.testDiscovery.refresh(); + void this.testDiscovery.refresh(); } private handleDidChangeTests(): void { @@ -209,7 +209,7 @@ export class TestManager extends TestManagerBase { */ public updateTestsForWorkspaceFolder( workspaceFolder: WorkspaceFolder, - testDirectory: QLTestDirectory | undefined, + testDirectory: FileTreeDirectory | undefined, ): void { if (testDirectory !== undefined) { // Adding an item with the same ID as an existing item will replace it, which is exactly what @@ -229,9 +229,9 @@ export class TestManager extends TestManagerBase { /** * Creates a tree of `TestItem`s from the root `QlTestNode` provided by test discovery. */ - private createTestItemTree(node: QLTestNode, isRoot: boolean): TestItem { + private createTestItemTree(node: FileTreeNode, isRoot: boolean): TestItem { // Prefix the ID to identify it as a directory or a test - const itemType = node instanceof QLTestDirectory ? "dir" : "test"; + const itemType = node instanceof FileTreeDirectory ? "dir" : "test"; const testItem = this.testController.createTestItem( // For the root of a workspace folder, use the full path as the ID. Otherwise, use the node's // name as the ID, since it's shorter but still unique. @@ -242,7 +242,7 @@ export class TestManager extends TestManagerBase { for (const childNode of node.children) { const childItem = this.createTestItemTree(childNode, false); - if (childNode instanceof QLTestFile) { + if (childNode instanceof FileTreeLeaf) { childItem.range = new Range(0, 0, 0, 0); } testItem.children.add(childItem); diff --git a/extensions/ql-vscode/src/run-queries-shared.ts b/extensions/ql-vscode/src/run-queries-shared.ts index cdb472a0b..811fe57df 100644 --- a/extensions/ql-vscode/src/run-queries-shared.ts +++ b/extensions/ql-vscode/src/run-queries-shared.ts @@ -433,6 +433,7 @@ export function validateQueryPath( export interface QuickEvalContext { quickEvalPosition: messages.Position; quickEvalText: string; + quickEvalCount: boolean; } /** @@ -443,6 +444,7 @@ export interface QuickEvalContext { */ export async function getQuickEvalContext( range: Range | undefined, + isCountOnly: boolean, ): Promise { const editor = window.activeTextEditor; if (editor === undefined) { @@ -465,6 +467,7 @@ export async function getQuickEvalContext( return { quickEvalPosition, quickEvalText, + quickEvalCount: isCountOnly, }; } diff --git a/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx b/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx index 922aa38fa..789c0ff06 100644 --- a/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx +++ b/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx @@ -30,6 +30,7 @@ DataExtensionsEditor.args = { "/home/user/vscode-codeql-starter/codeql-custom-queries-java/sql2o/models/sql2o.yml", }, modelFileExists: true, + showLlmButton: true, }, initialExternalApiUsages: [ { diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 00bbafde9..0e8d68d1f 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -7,6 +7,43 @@ import { VariantAnalysisSubmissionRequest, } from "./variant-analysis"; import { Repository } from "./repository"; +import { Progress } from "vscode"; +import { CancellationToken } from "vscode-jsonrpc"; + +export async function getCodeSearchRepositories( + credentials: Credentials, + query: string, + progress: Progress<{ + message?: string | undefined; + increment?: number | undefined; + }>, + token: CancellationToken, +): Promise { + let nwos: string[] = []; + const octokit = await credentials.getOctokit(); + for await (const response of octokit.paginate.iterator( + octokit.rest.search.repos, + { + q: query, + per_page: 100, + }, + )) { + nwos.push(...response.data.map((item) => item.full_name)); + // calculate progress bar: 80% of the progress bar is used for the code search + const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); + // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10 + const increment = + totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; + progress.report({ increment }); + + if (token.isCancellationRequested) { + nwos = []; + break; + } + } + + return [...new Set(nwos)]; +} export async function submitVariantAnalysis( credentials: Credentials, diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts index aa6285cfe..a45418e3b 100644 --- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts +++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts @@ -116,12 +116,16 @@ async function generateQueryPack( let precompilationOpts: string[] = []; if (await cliServer.cliConstraints.supportsQlxRemote()) { - const ccache = join(originalPackRoot, ".cache"); - precompilationOpts = [ - "--qlx", - "--no-default-compilation-cache", - `--compilation-cache=${ccache}`, - ]; + if (await cliServer.cliConstraints.usesGlobalCompilationCache()) { + precompilationOpts = ["--qlx"]; + } else { + const ccache = join(originalPackRoot, ".cache"); + precompilationOpts = [ + "--qlx", + "--no-default-compilation-cache", + `--compilation-cache=${ccache}`, + ]; + } } else { precompilationOpts = ["--no-precompile"]; } @@ -379,7 +383,6 @@ async function fixPackFile( } const qlpack = load(await readFile(packPath, "utf8")) as QlPack; - qlpack.name = QUERY_PACK_NAME; updateDefaultSuite(qlpack, packRelativePath); removeWorkspaceRefs(qlpack); diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts index f639d5b7d..6c84b9915 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts @@ -5,6 +5,8 @@ import { getVariantAnalysisRepo, } from "./gh-api/gh-api-client"; import { + authentication, + AuthenticationSessionsChangeEvent, CancellationToken, env, EventEmitter, @@ -72,6 +74,11 @@ import { REPO_STATES_FILENAME, writeRepoStates, } from "./repo-states-store"; +import { GITHUB_AUTH_PROVIDER_ID } from "../common/vscode/authentication"; +import { FetchError } from "node-fetch"; +import { extLogger } from "../common"; + +const maxRetryCount = 3; export class VariantAnalysisManager extends DisposableObject @@ -131,6 +138,10 @@ export class VariantAnalysisManager this.variantAnalysisResultsManager.onResultLoaded( this.onRepoResultLoaded.bind(this), ); + + this.push( + authentication.onDidChangeSessions(this.onDidChangeSessions.bind(this)), + ); } getCommands(): VariantAnalysisCommands { @@ -144,6 +155,8 @@ export class VariantAnalysisManager this.monitorVariantAnalysis.bind(this), "codeQL.monitorRehydratedVariantAnalysis": this.monitorVariantAnalysis.bind(this), + "codeQL.monitorReauthenticatedVariantAnalysis": + this.monitorVariantAnalysis.bind(this), "codeQL.openVariantAnalysisLogs": this.openVariantAnalysisLogs.bind(this), "codeQL.openVariantAnalysisView": this.showView.bind(this), "codeQL.runVariantAnalysis": @@ -504,6 +517,38 @@ export class VariantAnalysisManager repoStates[repoState.repositoryId] = repoState; } + private async onDidChangeSessions( + event: AuthenticationSessionsChangeEvent, + ): Promise { + if (event.provider.id !== GITHUB_AUTH_PROVIDER_ID) { + return; + } + + for (const variantAnalysis of this.variantAnalyses.values()) { + if ( + this.variantAnalysisMonitor.isMonitoringVariantAnalysis( + variantAnalysis.id, + ) + ) { + continue; + } + + if ( + await isVariantAnalysisComplete( + variantAnalysis, + this.makeResultDownloadChecker(variantAnalysis), + ) + ) { + continue; + } + + void this.app.commands.execute( + "codeQL.monitorReauthenticatedVariantAnalysis", + variantAnalysis, + ); + } + } + public async monitorVariantAnalysis( variantAnalysis: VariantAnalysis, ): Promise { @@ -572,12 +617,35 @@ export class VariantAnalysisManager }); } }; - await this.variantAnalysisResultsManager.download( - variantAnalysis.id, - repoTask, - this.getVariantAnalysisStorageLocation(variantAnalysis.id), - updateRepoStateCallback, - ); + let retry = 0; + for (;;) { + try { + await this.variantAnalysisResultsManager.download( + variantAnalysis.id, + repoTask, + this.getVariantAnalysisStorageLocation(variantAnalysis.id), + updateRepoStateCallback, + ); + break; + } catch (e) { + if ( + retry++ < maxRetryCount && + e instanceof FetchError && + (e.code === "ETIMEDOUT" || e.code === "ECONNRESET") + ) { + void extLogger.log( + `Timeout while trying to download variant analysis with id: ${ + variantAnalysis.id + }. Error: ${getErrorMessage(e)}. Retrying...`, + ); + continue; + } + void extLogger.log( + `Failed to download variant analysis after ${retry} attempts.`, + ); + throw e; + } + } } catch (e) { repoState.downloadStatus = VariantAnalysisScannedRepositoryDownloadStatus.Failed; diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts index c3b48a9db..32ae083d4 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts @@ -1,5 +1,6 @@ import { env, EventEmitter } from "vscode"; import { getVariantAnalysis } from "./gh-api/gh-api-client"; +import { RequestError } from "@octokit/request-error"; import { isFinalVariantAnalysisStatus, @@ -27,6 +28,8 @@ export class VariantAnalysisMonitor extends DisposableObject { ); readonly onVariantAnalysisChange = this._onVariantAnalysisChange.event; + private readonly monitoringVariantAnalyses = new Set(); + constructor( private readonly app: App, private readonly shouldCancelMonitor: ( @@ -36,9 +39,37 @@ export class VariantAnalysisMonitor extends DisposableObject { super(); } + public isMonitoringVariantAnalysis(variantAnalysisId: number): boolean { + return this.monitoringVariantAnalyses.has(variantAnalysisId); + } + public async monitorVariantAnalysis( variantAnalysis: VariantAnalysis, ): Promise { + if (this.monitoringVariantAnalyses.has(variantAnalysis.id)) { + void extLogger.log( + `Already monitoring variant analysis ${variantAnalysis.id}`, + ); + return; + } + + this.monitoringVariantAnalyses.add(variantAnalysis.id); + try { + await this._monitorVariantAnalysis(variantAnalysis); + } finally { + this.monitoringVariantAnalyses.delete(variantAnalysis.id); + } + } + + private async _monitorVariantAnalysis( + variantAnalysis: VariantAnalysis, + ): Promise { + const variantAnalysisLabel = `${variantAnalysis.query.name} (${ + variantAnalysis.query.language + }) [${new Date(variantAnalysis.executionStartTime).toLocaleString( + env.language, + )}]`; + let attemptCount = 0; const scannedReposDownloaded: number[] = []; @@ -61,11 +92,7 @@ export class VariantAnalysisMonitor extends DisposableObject { } catch (e) { const errorMessage = getErrorMessage(e); - const message = `Error while monitoring variant analysis ${ - variantAnalysis.query.name - } (${variantAnalysis.query.language}) [${new Date( - variantAnalysis.executionStartTime, - ).toLocaleString(env.language)}]: ${errorMessage}`; + const message = `Error while monitoring variant analysis ${variantAnalysisLabel}: ${errorMessage}`; // If we have already shown this error to the user, don't show it again. if (lastErrorShown === errorMessage) { @@ -75,6 +102,19 @@ export class VariantAnalysisMonitor extends DisposableObject { lastErrorShown = errorMessage; } + if (e instanceof RequestError && e.status === 404) { + // We want to show the error message to the user, but we don't want to + // keep polling for the variant analysis if it no longer exists. + // Therefore, this block is down here rather than at the top of the + // catch block. + void extLogger.log( + `Variant analysis ${variantAnalysisLabel} no longer exists or is no longer accessible, stopping monitoring.`, + ); + // Cancel monitoring on 404, as this probably means the user does not have access to it anymore + // e.g. lost access to repo, or repo was deleted + return; + } + continue; } diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts index 5c538e621..2ccbee55d 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts @@ -1,4 +1,4 @@ -import { appendFile, pathExists } from "fs-extra"; +import { appendFile, pathExists, rm } from "fs-extra"; import fetch from "node-fetch"; import { EOL } from "os"; import { join } from "path"; @@ -82,6 +82,9 @@ export class VariantAnalysisResultsManager extends DisposableObject { const zipFilePath = join(resultDirectory, "results.zip"); + // in case of restarted download delete possible artifact from previous download + await rm(zipFilePath, { force: true }); + const response = await fetch(repoTask.artifactUrl); let responseSize = parseInt(response.headers.get("content-length") || "0"); diff --git a/extensions/ql-vscode/src/view/compare/Compare.tsx b/extensions/ql-vscode/src/view/compare/Compare.tsx index 00aef5449..a98626e42 100644 --- a/extensions/ql-vscode/src/view/compare/Compare.tsx +++ b/extensions/ql-vscode/src/view/compare/Compare.tsx @@ -59,9 +59,7 @@ export function Compare(_: Record): JSX.Element { return ( <>
-
- Table to compare: -
+
Comparing:
props.updateResultSet(e.target.value)} @@ -18,5 +19,8 @@ export default function CompareSelector(props: Props) { ))} + ) : ( + // Handle case where there are no shared result sets +
{props.currentResultSetName}
); } diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index 592625e5f..4b09a8424 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -157,6 +157,14 @@ export function DataExtensionsEditor({ }); }, []); + const onGenerateFromLlmClick = useCallback(() => { + vscode.postMessage({ + t: "generateExternalApiFromLlm", + externalApiUsages, + modeledMethods, + }); + }, [externalApiUsages, modeledMethods]); + const onOpenExtensionPackClick = useCallback(() => { vscode.postMessage({ t: "openExtensionPack", @@ -214,6 +222,14 @@ export function DataExtensionsEditor({ Download and generate + {viewState?.showLlmButton && ( + <> +   + + Generate using LLM + + + )}

diff --git a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx index 4df2a9157..26ce0b3e7 100644 --- a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx +++ b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx @@ -14,6 +14,9 @@ export type VariantAnalysisActionsProps = { onExportResultsClick: () => void; copyRepositoryListDisabled?: boolean; exportResultsDisabled?: boolean; + + hasSelectedRepositories?: boolean; + hasFilteredRepositories?: boolean; }; const Container = styled.div` @@ -26,6 +29,28 @@ const Button = styled(VSCodeButton)` white-space: nowrap; `; +const chooseText = ({ + hasSelectedRepositories, + hasFilteredRepositories, + normalText, + selectedText, + filteredText, +}: { + hasSelectedRepositories?: boolean; + hasFilteredRepositories?: boolean; + normalText: string; + selectedText: string; + filteredText: string; +}) => { + if (hasSelectedRepositories) { + return selectedText; + } + if (hasFilteredRepositories) { + return filteredText; + } + return normalText; +}; + export const VariantAnalysisActions = ({ variantAnalysisStatus, onStopQueryClick, @@ -35,6 +60,8 @@ export const VariantAnalysisActions = ({ onExportResultsClick, copyRepositoryListDisabled, exportResultsDisabled, + hasSelectedRepositories, + hasFilteredRepositories, }: VariantAnalysisActionsProps) => { return ( @@ -45,14 +72,26 @@ export const VariantAnalysisActions = ({ onClick={onCopyRepositoryListClick} disabled={copyRepositoryListDisabled} > - Copy repository list + {chooseText({ + hasSelectedRepositories, + hasFilteredRepositories, + normalText: "Copy repository list", + selectedText: "Copy selected repositories as a list", + filteredText: "Copy filtered repositories as a list", + })} )} diff --git a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx index a579a4268..1c146f19f 100644 --- a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx +++ b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx @@ -131,6 +131,13 @@ export const VariantAnalysisHeader = ({ stopQueryDisabled={!variantAnalysis.actionsWorkflowRunId} exportResultsDisabled={!hasDownloadedRepos} copyRepositoryListDisabled={!hasReposWithResults} + hasFilteredRepositories={ + variantAnalysis.scannedRepos?.length !== + filteredRepositories?.length + } + hasSelectedRepositories={ + selectedRepositoryIds && selectedRepositoryIds.length > 0 + } /> { expect(container.querySelectorAll("vscode-button").length).toEqual(0); }); + + it("changes the text on the buttons when repositories are selected", async () => { + render({ + variantAnalysisStatus: VariantAnalysisStatus.Succeeded, + showResultActions: true, + hasSelectedRepositories: true, + hasFilteredRepositories: true, + }); + + expect(screen.getByText("Export selected results")).toBeInTheDocument(); + expect( + screen.getByText("Copy selected repositories as a list"), + ).toBeInTheDocument(); + }); + + it("changes the text on the buttons when repositories are filtered", async () => { + render({ + variantAnalysisStatus: VariantAnalysisStatus.Succeeded, + showResultActions: true, + hasSelectedRepositories: false, + hasFilteredRepositories: true, + }); + + expect(screen.getByText("Export filtered results")).toBeInTheDocument(); + expect( + screen.getByText("Copy filtered repositories as a list"), + ).toBeInTheDocument(); + }); }); diff --git a/extensions/ql-vscode/supported_cli_versions.json b/extensions/ql-vscode/supported_cli_versions.json index 7fa2cc8c1..fe9f952c6 100644 --- a/extensions/ql-vscode/supported_cli_versions.json +++ b/extensions/ql-vscode/supported_cli_versions.json @@ -1,5 +1,5 @@ [ - "v2.13.1", + "v2.13.3", "v2.12.7", "v2.11.6", "v2.7.6", diff --git a/extensions/ql-vscode/test/__mocks__/appMock.ts b/extensions/ql-vscode/test/__mocks__/appMock.ts index 1b6d9da9d..418310d5b 100644 --- a/extensions/ql-vscode/test/__mocks__/appMock.ts +++ b/extensions/ql-vscode/test/__mocks__/appMock.ts @@ -8,6 +8,11 @@ import { testCredentialsWithStub } from "../factories/authentication"; import { Credentials } from "../../src/common/authentication"; import { AppCommandManager } from "../../src/common/commands"; import { createMockCommandManager } from "./commandsMock"; +import type { + Event, + WorkspaceFolder, + WorkspaceFoldersChangeEvent, +} from "vscode"; export function createMockApp({ extensionPath = "/mock/extension/path", @@ -15,6 +20,8 @@ export function createMockApp({ globalStoragePath = "/mock/global/storage/path", createEventEmitter = () => new MockAppEventEmitter(), workspaceState = createMockMemento(), + workspaceFolders = [], + onDidChangeWorkspaceFolders = jest.fn(), credentials = testCredentialsWithStub(), commands = createMockCommandManager(), }: { @@ -23,6 +30,8 @@ export function createMockApp({ globalStoragePath?: string; createEventEmitter?: () => AppEventEmitter; workspaceState?: Memento; + workspaceFolders?: readonly WorkspaceFolder[] | undefined; + onDidChangeWorkspaceFolders?: Event; credentials?: Credentials; commands?: AppCommandManager; }): App { @@ -34,6 +43,8 @@ export function createMockApp({ workspaceStoragePath, globalStoragePath, workspaceState, + workspaceFolders, + onDidChangeWorkspaceFolders, createEventEmitter, credentials, commands, @@ -52,4 +63,8 @@ export class MockAppEventEmitter implements AppEventEmitter { public fire(): void { // no-op } + + public dispose() { + // no-op + } } diff --git a/extensions/ql-vscode/test/factories/databases/databases.ts b/extensions/ql-vscode/test/factories/databases/databases.ts index f6aa2aebf..46826917a 100644 --- a/extensions/ql-vscode/test/factories/databases/databases.ts +++ b/extensions/ql-vscode/test/factories/databases/databases.ts @@ -33,7 +33,6 @@ export function createMockDB( datasetUri: databaseUri, } as DatabaseContents, dbOptions, - () => void 0, ); } diff --git a/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts b/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts new file mode 100644 index 000000000..4623c62a5 --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts @@ -0,0 +1,143 @@ +import type { Memento } from "vscode"; +import { InvocationRateLimiter } from "../../../src/common/invocation-rate-limiter"; + +describe("Invocation rate limiter", () => { + // 1 January 2020 + let currentUnixTime = 1577836800; + + function createDate(dateString?: string): Date { + if (dateString) { + return new Date(dateString); + } + const numMillisecondsPerSecond = 1000; + return new Date(currentUnixTime * numMillisecondsPerSecond); + } + + function createInvocationRateLimiter( + funcIdentifier: string, + func: () => Promise, + ): InvocationRateLimiter { + return new InvocationRateLimiter( + new MockMemento(), + funcIdentifier, + func, + (s) => createDate(s), + ); + } + + class MockMemento implements Memento { + keys(): readonly string[] { + throw new Error("Method not implemented."); + } + map = new Map(); + + /** + * Return a value. + * + * @param key A string. + * @param defaultValue A value that should be returned when there is no + * value (`undefined`) with the given key. + * @return The stored value or the defaultValue. + */ + get(key: string, defaultValue?: T): T { + return this.map.has(key) ? this.map.get(key) : defaultValue; + } + + /** + * Store a value. The value must be JSON-stringifyable. + * + * @param key A string. + * @param value A value. MUST not contain cyclic references. + */ + async update(key: string, value: any): Promise { + this.map.set(key, value); + } + } + + it("initially invokes function", async () => { + let numTimesFuncCalled = 0; + const invocationRateLimiter = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncCalled++; + }, + ); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); + expect(numTimesFuncCalled).toBe(1); + }); + + it("doesn't invoke function again if no time has passed", async () => { + let numTimesFuncCalled = 0; + const invocationRateLimiter = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncCalled++; + }, + ); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); + expect(numTimesFuncCalled).toBe(1); + }); + + it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => { + let numTimesFuncCalled = 0; + const invocationRateLimiter = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncCalled++; + }, + ); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); + currentUnixTime += 1; + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2); + expect(numTimesFuncCalled).toBe(1); + }); + + it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => { + let numTimesFuncCalled = 0; + const invocationRateLimiter = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncCalled++; + }, + ); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0); + expect(numTimesFuncCalled).toBe(2); + }); + + it("invokes function again after requested time since last invocation has elapsed", async () => { + let numTimesFuncCalled = 0; + const invocationRateLimiter = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncCalled++; + }, + ); + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1); + currentUnixTime += 1; + await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1); + expect(numTimesFuncCalled).toBe(2); + }); + + it("invokes functions with different rate limiters", async () => { + let numTimesFuncACalled = 0; + const invocationRateLimiterA = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncACalled++; + }, + ); + let numTimesFuncBCalled = 0; + const invocationRateLimiterB = createInvocationRateLimiter( + "funcid", + async () => { + numTimesFuncBCalled++; + }, + ); + await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100); + await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100); + expect(numTimesFuncACalled).toBe(1); + expect(numTimesFuncBCalled).toBe(1); + }); +}); diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts new file mode 100644 index 000000000..c2f372f01 --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts @@ -0,0 +1,471 @@ +import { + compareInputOutput, + createAutoModelRequest, + parsePredictedClassifications, +} from "../../../src/data-extensions-editor/auto-model"; +import { ExternalApiUsage } from "../../../src/data-extensions-editor/external-api-usage"; +import { ModeledMethod } from "../../../src/data-extensions-editor/modeled-method"; +import { + ClassificationType, + Method, +} from "../../../src/data-extensions-editor/auto-model-api"; + +describe("createAutoModelRequest", () => { + const externalApiUsages: ExternalApiUsage[] = [ + { + signature: + "org.springframework.boot.SpringApplication#run(Class,String[])", + packageName: "org.springframework.boot", + typeName: "SpringApplication", + methodName: "run", + methodParameters: "(Class,String[])", + supported: false, + usages: [ + { + label: "run(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java", + startLine: 9, + startColumn: 9, + endLine: 9, + endColumn: 66, + }, + }, + ], + }, + { + signature: "org.sql2o.Connection#createQuery(String)", + packageName: "org.sql2o", + typeName: "Connection", + methodName: "createQuery", + methodParameters: "(String)", + supported: true, + usages: [ + { + label: "createQuery(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 15, + startColumn: 13, + endLine: 15, + endColumn: 56, + }, + }, + { + label: "createQuery(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 26, + startColumn: 13, + endLine: 26, + endColumn: 39, + }, + }, + ], + }, + { + signature: "org.sql2o.Query#executeScalar(Class)", + packageName: "org.sql2o", + typeName: "Query", + methodName: "executeScalar", + methodParameters: "(Class)", + supported: true, + usages: [ + { + label: "executeScalar(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 15, + startColumn: 13, + endLine: 15, + endColumn: 85, + }, + }, + { + label: "executeScalar(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 26, + startColumn: 13, + endLine: 26, + endColumn: 68, + }, + }, + ], + }, + { + signature: "org.sql2o.Sql2o#open()", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "open", + methodParameters: "()", + supported: true, + usages: [ + { + label: "open(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 14, + startColumn: 24, + endLine: 14, + endColumn: 35, + }, + }, + { + label: "open(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 25, + startColumn: 24, + endLine: 25, + endColumn: 35, + }, + }, + ], + }, + { + signature: "java.io.PrintStream#println(String)", + packageName: "java.io", + typeName: "PrintStream", + methodName: "println", + methodParameters: "(String)", + supported: true, + usages: [ + { + label: "println(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 29, + startColumn: 9, + endLine: 29, + endColumn: 49, + }, + }, + ], + }, + { + signature: "org.sql2o.Sql2o#Sql2o(String,String,String)", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "Sql2o", + methodParameters: "(String,String,String)", + supported: true, + usages: [ + { + label: "new Sql2o(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 10, + startColumn: 33, + endLine: 10, + endColumn: 88, + }, + }, + ], + }, + { + signature: "org.sql2o.Sql2o#Sql2o(String)", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "Sql2o", + methodParameters: "(String)", + supported: true, + usages: [ + { + label: "new Sql2o(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 23, + startColumn: 23, + endLine: 23, + endColumn: 36, + }, + }, + ], + }, + ]; + + const modeledMethods: Record = { + "org.sql2o.Sql2o#open()": { + type: "neutral", + kind: "", + input: "", + output: "", + }, + "org.sql2o.Sql2o#Sql2o(String)": { + type: "sink", + kind: "jndi-injection", + input: "Argument[0]", + output: "", + }, + }; + + const usages: Record = { + "org.springframework.boot.SpringApplication#run(Class,String[])": [ + "public class Sql2oExampleApplication {\n public static void main(String[] args) {\n SpringApplication.run(Sql2oExampleApplication.class, args);\n }\n}", + ], + "org.sql2o.Connection#createQuery(String)": [ + ' public String index(@RequestParam("id") String id) {\n try (var con = sql2o.open()) {\n con.createQuery("select 1 where id = " + id).executeScalar(Integer.class);\n }\n\n', + '\n try (var con = sql2o.open()) {\n con.createQuery("select 1").executeScalar(Integer.class);\n }\n\n', + ], + "org.sql2o.Query#executeScalar(Class)": [ + ' public String index(@RequestParam("id") String id) {\n try (var con = sql2o.open()) {\n con.createQuery("select 1 where id = " + id).executeScalar(Integer.class);\n }\n\n', + '\n try (var con = sql2o.open()) {\n con.createQuery("select 1").executeScalar(Integer.class);\n }\n\n', + ], + "org.sql2o.Sql2o#open()": [ + ' @GetMapping("/")\n public String index(@RequestParam("id") String id) {\n try (var con = sql2o.open()) {\n con.createQuery("select 1 where id = " + id).executeScalar(Integer.class);\n }\n', + ' Sql2o sql2o = new Sql2o(url);\n\n try (var con = sql2o.open()) {\n con.createQuery("select 1").executeScalar(Integer.class);\n }\n', + ], + "java.io.PrintStream#println(String)": [ + ' }\n\n System.out.println("Connected to " + url);\n\n return "Greetings from Spring Boot!";\n', + ], + "org.sql2o.Sql2o#Sql2o(String,String,String)": [ + '@RestController\npublic class HelloController {\n private final Sql2o sql2o = new Sql2o("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1","sa", "");\n\n @GetMapping("/")\n', + ], + "org.sql2o.Sql2o#Sql2o(String)": [ + ' @GetMapping("/connect")\n public String connect(@RequestParam("url") String url) {\n Sql2o sql2o = new Sql2o(url);\n\n try (var con = sql2o.open()) {\n', + ], + }; + + it("creates a matching request", () => { + expect( + createAutoModelRequest("java", externalApiUsages, modeledMethods, usages), + ).toEqual({ + language: "java", + samples: [ + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String)", + classification: { + type: "CLASSIFICATION_TYPE_SINK", + kind: "jndi-injection", + explanation: "", + }, + usages: usages["org.sql2o.Sql2o#Sql2o(String)"], + input: "Argument[0]", + }, + ], + candidates: [ + { + package: "org.sql2o", + type: "Connection", + name: "createQuery", + signature: "(String)", + usages: usages["org.sql2o.Connection#createQuery(String)"], + input: "Argument[0]", + classification: undefined, + }, + { + package: "org.sql2o", + type: "Query", + name: "executeScalar", + signature: "(Class)", + usages: usages["org.sql2o.Query#executeScalar(Class)"], + input: "Argument[0]", + classification: undefined, + }, + { + package: "org.springframework.boot", + type: "SpringApplication", + name: "run", + signature: "(Class,String[])", + usages: + usages[ + "org.springframework.boot.SpringApplication#run(Class,String[])" + ], + input: "Argument[0]", + classification: undefined, + }, + { + package: "org.springframework.boot", + type: "SpringApplication", + name: "run", + signature: "(Class,String[])", + usages: + usages[ + "org.springframework.boot.SpringApplication#run(Class,String[])" + ], + input: "Argument[1]", + classification: undefined, + }, + { + package: "java.io", + type: "PrintStream", + name: "println", + signature: "(String)", + usages: usages["java.io.PrintStream#println(String)"], + input: "Argument[0]", + classification: undefined, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String,String,String)", + usages: usages["org.sql2o.Sql2o#Sql2o(String,String,String)"], + input: "Argument[0]", + classification: undefined, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String,String,String)", + usages: usages["org.sql2o.Sql2o#Sql2o(String,String,String)"], + input: "Argument[1]", + classification: undefined, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String,String,String)", + usages: usages["org.sql2o.Sql2o#Sql2o(String,String,String)"], + input: "Argument[2]", + classification: undefined, + }, + ], + }); + }); +}); + +describe("parsePredictedClassifications", () => { + const predictions: Method[] = [ + { + package: "org.sql2o", + type: "Sql2o", + name: "createQuery", + signature: "(String)", + usages: ["createQuery(...)", "createQuery(...)"], + input: "Argument[0]", + classification: { + type: ClassificationType.Sink, + kind: "sql injection sink", + explanation: "", + }, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "executeScalar", + signature: "(Class)", + usages: ["executeScalar(...)", "executeScalar(...)"], + input: "Argument[0]", + classification: { + type: ClassificationType.Neutral, + kind: "", + explanation: "not a sink", + }, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String,String,String)", + usages: ["new Sql2o(...)"], + input: "Argument[0]", + classification: { + type: ClassificationType.Neutral, + kind: "", + explanation: "not a sink", + }, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String,String,String)", + usages: ["new Sql2o(...)"], + input: "Argument[1]", + classification: { + type: ClassificationType.Sink, + kind: "sql injection sink", + explanation: "", + }, + }, + { + package: "org.sql2o", + type: "Sql2o", + name: "Sql2o", + signature: "(String,String,String)", + usages: ["new Sql2o(...)"], + input: "Argument[2]", + classification: { + type: ClassificationType.Sink, + kind: "sql injection sink", + explanation: "", + }, + }, + ]; + + it("correctly parses the output", () => { + expect(parsePredictedClassifications(predictions)).toEqual({ + "org.sql2o.Sql2o#createQuery(String)": { + type: "sink", + kind: "sql injection sink", + input: "Argument[0]", + output: "", + }, + "org.sql2o.Sql2o#executeScalar(Class)": { + type: "neutral", + kind: "", + input: "", + output: "", + }, + "org.sql2o.Sql2o#Sql2o(String,String,String)": { + type: "sink", + kind: "sql injection sink", + input: "Argument[1]", + output: "", + }, + }); + }); +}); + +describe("compareInputOutput", () => { + it("with two small numeric arguments", () => { + expect( + compareInputOutput("Argument[0]", "Argument[1]"), + ).toBeLessThanOrEqual(-1); + }); + + it("with one larger non-alphabetic argument", () => { + expect( + compareInputOutput("Argument[10]", "Argument[2]"), + ).toBeGreaterThanOrEqual(1); + }); + + it("with one non-numeric arguments", () => { + expect( + compareInputOutput("Argument[5]", "Argument[this]"), + ).toBeLessThanOrEqual(-1); + }); + + it("with two non-numeric arguments", () => { + expect( + compareInputOutput("ReturnValue", "Argument[this]"), + ).toBeGreaterThanOrEqual(1); + }); + + it("with one unknown argument in the a position", () => { + expect( + compareInputOutput("FooBar", "Argument[this]"), + ).toBeGreaterThanOrEqual(1); + }); + + it("with one unknown argument in the b position", () => { + expect(compareInputOutput("Argument[this]", "FooBar")).toBeLessThanOrEqual( + -1, + ); + }); + + it("with one empty string arguments", () => { + expect(compareInputOutput("Argument[5]", "")).toBeLessThanOrEqual(-1); + }); + + it("with two unknown arguments", () => { + expect(compareInputOutput("FooBar", "BarFoo")).toBeGreaterThanOrEqual(1); + }); +}); diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts index 99317ceda..6d22b178e 100644 --- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts +++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts @@ -4,14 +4,13 @@ import { DecodedBqrsChunk } from "../../../src/pure/bqrs-cli-types"; describe("decodeBqrsToExternalApiUsages", () => { const chunk: DecodedBqrsChunk = { columns: [ - { name: "apiName", kind: "String" }, - { name: "supported", kind: "Boolean" }, { name: "usage", kind: "Entity" }, + { name: "apiName", kind: "String" }, + { kind: "String" }, + { kind: "String" }, ], tuples: [ [ - "java.io.PrintStream#println(String)", - true, { label: "println(...)", url: { @@ -22,10 +21,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 49, }, }, + "java.io.PrintStream#println(String)", + "true", + "supported", ], [ - "org.springframework.boot.SpringApplication#run(Class,String[])", - false, { label: "run(...)", url: { @@ -36,10 +36,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 66, }, }, + "org.springframework.boot.SpringApplication#run(Class,String[])", + "false", + "supported", ], [ - "org.sql2o.Connection#createQuery(String)", - true, { label: "createQuery(...)", url: { @@ -50,10 +51,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 56, }, }, + "org.sql2o.Connection#createQuery(String)", + "true", + "supported", ], [ - "org.sql2o.Connection#createQuery(String)", - true, { label: "createQuery(...)", url: { @@ -64,10 +66,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 39, }, }, + "org.sql2o.Connection#createQuery(String)", + "true", + "supported", ], [ - "org.sql2o.Query#executeScalar(Class)", - true, { label: "executeScalar(...)", url: { @@ -78,10 +81,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 85, }, }, + "org.sql2o.Query#executeScalar(Class)", + "true", + "supported", ], [ - "org.sql2o.Query#executeScalar(Class)", - true, { label: "executeScalar(...)", url: { @@ -92,10 +96,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 68, }, }, + "org.sql2o.Query#executeScalar(Class)", + "true", + "supported", ], [ - "org.sql2o.Sql2o#open()", - true, { label: "open(...)", url: { @@ -106,10 +111,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 35, }, }, + "org.sql2o.Sql2o#open()", + "true", + "supported", ], [ - "org.sql2o.Sql2o#open()", - true, { label: "open(...)", url: { @@ -120,10 +126,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 35, }, }, + "org.sql2o.Sql2o#open()", + "true", + "supported", ], [ - "org.sql2o.Sql2o#Sql2o(String,String,String)", - true, { label: "new Sql2o(...)", url: { @@ -134,10 +141,11 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 88, }, }, + "org.sql2o.Sql2o#Sql2o(String,String,String)", + "true", + "supported", ], [ - "org.sql2o.Sql2o#Sql2o(String)", - true, { label: "new Sql2o(...)", url: { @@ -148,6 +156,9 @@ describe("decodeBqrsToExternalApiUsages", () => { endColumn: 36, }, }, + "org.sql2o.Sql2o#Sql2o(String)", + "true", + "supported", ], ], }; diff --git a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts index ca4b7ead6..b7a3fd997 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts @@ -241,6 +241,113 @@ describe("db config store", () => { configStore.dispose(); }); + it("should add unique remote repositories to the correct list", async () => { + // Initial set up + const dbConfig = createDbConfig({ + remoteLists: [ + { + name: "list1", + repositories: ["owner/repo1"], + }, + ], + }); + + const configStore = await initializeConfig(dbConfig, configPath, app); + expect( + configStore.getConfig().value.databases.variantAnalysis + .repositoryLists[0], + ).toEqual({ + name: "list1", + repositories: ["owner/repo1"], + }); + + // Add + const response = await configStore.addRemoteReposToList( + ["owner/repo1", "owner/repo2"], + "list1", + ); + + // Read the config file + const updatedDbConfig = (await readJSON(configPath)) as DbConfig; + + // Check that the config file has been updated + const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis; + expect(updatedRemoteDbs.repositories).toHaveLength(0); + expect(updatedRemoteDbs.repositoryLists).toHaveLength(1); + expect(updatedRemoteDbs.repositoryLists[0]).toEqual({ + name: "list1", + repositories: ["owner/repo1", "owner/repo2"], + }); + expect(response).toEqual([]); + + configStore.dispose(); + }); + + it("should add no more than 1000 repositories to a remote list when adding multiple repos", async () => { + // Initial set up + const dbConfig = createDbConfig({ + remoteLists: [ + { + name: "list1", + repositories: [], + }, + ], + }); + + const configStore = await initializeConfig(dbConfig, configPath, app); + + // Add + const response = await configStore.addRemoteReposToList( + [...Array(1001).keys()].map((i) => `owner/db${i}`), + "list1", + ); + + // Read the config file + const updatedDbConfig = (await readJSON(configPath)) as DbConfig; + + // Check that the config file has been updated + const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis; + expect(updatedRemoteDbs.repositories).toHaveLength(0); + expect(updatedRemoteDbs.repositoryLists).toHaveLength(1); + expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength( + 1000, + ); + expect(response).toEqual(["owner/db1000"]); + + configStore.dispose(); + }); + + it("should add no more than 1000 repositories to a remote list when adding one repo", async () => { + // Initial set up + const dbConfig = createDbConfig({ + remoteLists: [ + { + name: "list1", + repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`), + }, + ], + }); + + const configStore = await initializeConfig(dbConfig, configPath, app); + + // Add + const reponse = await configStore.addRemoteRepo("owner/db1000", "list1"); + + // Read the config file + const updatedDbConfig = (await readJSON(configPath)) as DbConfig; + + // Check that the config file has been updated + const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis; + expect(updatedRemoteDbs.repositories).toHaveLength(0); + expect(updatedRemoteDbs.repositoryLists).toHaveLength(1); + expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength( + 1000, + ); + expect(reponse).toEqual(["owner/db1000"]); + + configStore.dispose(); + }); + it("should add a remote owner", async () => { // Initial set up const dbConfig = createDbConfig(); diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts index fa49ce2f9..60c87c356 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts @@ -88,6 +88,73 @@ describe("db manager", () => { ).toEqual("owner2/repo2"); }); + it("should add new remote repos to a user defined list", async () => { + const dbConfig: DbConfig = createDbConfig({ + remoteLists: [ + { + name: "my-list-1", + repositories: ["owner1/repo1"], + }, + ], + }); + + await saveDbConfig(dbConfig); + + await dbManager.addNewRemoteReposToList(["owner2/repo2"], "my-list-1"); + + const dbConfigFileContents = await readDbConfigDirectly(); + expect( + dbConfigFileContents.databases.variantAnalysis.repositoryLists.length, + ).toBe(1); + + expect( + dbConfigFileContents.databases.variantAnalysis.repositoryLists[0], + ).toEqual({ + name: "my-list-1", + repositories: ["owner1/repo1", "owner2/repo2"], + }); + }); + + it("should return truncated repos when adding multiple repos to a user defined list", async () => { + const dbConfig: DbConfig = createDbConfig({ + remoteLists: [ + { + name: "my-list-1", + repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`), + }, + ], + }); + + await saveDbConfig(dbConfig); + + const response = await dbManager.addNewRemoteReposToList( + ["owner2/repo2"], + "my-list-1", + ); + + expect(response).toEqual(["owner2/repo2"]); + }); + + it("should return truncated repos when adding one repo to a user defined list", async () => { + const dbConfig: DbConfig = createDbConfig({ + remoteLists: [ + { + name: "my-list-1", + repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`), + }, + ], + }); + + await saveDbConfig(dbConfig); + + const response = await dbManager.addNewRemoteRepo( + "owner2/repo2", + "my-list-1", + ); + + expect(response).toEqual(["owner2/repo2"]); + }); + it("should add a new remote repo to a user defined list", async () => { const dbConfig: DbConfig = createDbConfig({ remoteLists: [ diff --git a/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts b/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts index 036be67d8..48f8cfc83 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts @@ -62,12 +62,17 @@ describe("getDbItemActions", () => { expect(actions.length).toEqual(0); }); - it("should set canBeSelected, canBeRemoved and canBeRenamed for remote user defined db list", () => { + it("should set canBeSelected, canBeRemoved, canBeRenamed and canImportCodeSearch for remote user defined db list", () => { const dbItem = createRemoteUserDefinedListDbItem(); const actions = getDbItemActions(dbItem); - expect(actions).toEqual(["canBeSelected", "canBeRemoved", "canBeRenamed"]); + expect(actions).toEqual([ + "canBeSelected", + "canBeRemoved", + "canBeRenamed", + "canImportCodeSearch", + ]); }); it("should not set canBeSelected for remote user defined db list that is already selected", () => { diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts index b42e059df..9ffb9ee3a 100644 --- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts @@ -1,4 +1,5 @@ import * as ghApiClient from "../../../../src/variant-analysis/gh-api/gh-api-client"; +import { RequestError } from "@octokit/request-error"; import { VariantAnalysisMonitor } from "../../../../src/variant-analysis/variant-analysis-monitor"; import { VariantAnalysis as VariantAnalysisApiResponse, @@ -297,6 +298,55 @@ describe("Variant Analysis Monitor", () => { expect(mockEecuteCommand).not.toBeCalled(); }); }); + + describe("when a 404 is returned", () => { + let showAndLogWarningMessageSpy: jest.SpiedFunction< + typeof helpers.showAndLogWarningMessage + >; + + beforeEach(async () => { + showAndLogWarningMessageSpy = jest + .spyOn(helpers, "showAndLogWarningMessage") + .mockResolvedValue(undefined); + + const scannedRepos = createMockScannedRepos([ + "pending", + "in_progress", + "in_progress", + "in_progress", + "pending", + "pending", + ]); + mockApiResponse = createMockApiResponse("in_progress", scannedRepos); + mockGetVariantAnalysis.mockResolvedValueOnce(mockApiResponse); + + mockGetVariantAnalysis.mockRejectedValueOnce( + new RequestError("Not Found", 404, { + request: { + method: "GET", + url: "", + headers: {}, + }, + response: { + status: 404, + headers: {}, + url: "", + data: {}, + }, + }), + ); + }); + + it("should stop requesting the variant analysis", async () => { + await variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis); + + expect(mockGetVariantAnalysis).toHaveBeenCalledTimes(2); + expect(showAndLogWarningMessageSpy).toHaveBeenCalledTimes(1); + expect(showAndLogWarningMessageSpy).toHaveBeenCalledWith( + expect.stringMatching(/not found/i), + ); + }); + }); }); function limitNumberOfAttemptsToMonitor() { diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts index 491e6d037..d068a1295 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts @@ -28,7 +28,7 @@ import { QueryRunner, } from "../../../src/query-server/query-runner"; import { SELECT_QUERY_NAME } from "../../../src/language-support"; -import { LocalQueries } from "../../../src/local-queries"; +import { LocalQueries, QuickEvalType } from "../../../src/local-queries"; import { QueryResultType } from "../../../src/pure/new-messages"; import { createVSCodeCommandManager } from "../../../src/common/vscode/commands"; import { @@ -45,7 +45,7 @@ async function compileAndRunQuery( mode: DebugMode, appCommands: AppCommandManager, localQueries: LocalQueries, - quickEval: boolean, + quickEval: QuickEvalType, queryUri: Uri, progress: ProgressCallback, token: CancellationToken, @@ -184,7 +184,7 @@ describeWithCodeQL()("Queries", () => { mode, appCommandManager, localQueries, - false, + QuickEvalType.None, Uri.file(queryUsingExtensionPath), progress, token, @@ -218,7 +218,7 @@ describeWithCodeQL()("Queries", () => { mode, appCommandManager, localQueries, - false, + QuickEvalType.None, Uri.file(queryPath), progress, token, @@ -238,7 +238,7 @@ describeWithCodeQL()("Queries", () => { mode, appCommandManager, localQueries, - false, + QuickEvalType.None, Uri.file(queryPath), progress, token, diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts index 3bc1a6b85..71908530c 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts @@ -15,6 +15,7 @@ import { import { KeyType, resolveQueries } from "../../../src/language-support"; import { faker } from "@faker-js/faker"; import { getActivatedExtension } from "../global.helper"; +import { BaseLogger } from "../../../src/common"; /** * Perform proper integration tests by running the CLI @@ -23,10 +24,14 @@ describe("Use cli", () => { let cli: CodeQLCliServer; let supportedLanguages: string[]; + let logSpy: jest.SpiedFunction; + beforeEach(async () => { const extension = await getActivatedExtension(); cli = extension.cliServer; supportedLanguages = await cli.getSupportedLanguages(); + + logSpy = jest.spyOn(cli.logger, "log"); }); if (process.env.CLI_VERSION && process.env.CLI_VERSION !== "nightly") { @@ -42,6 +47,23 @@ describe("Use cli", () => { expect(result).toEqual(["-J-Xmx4096M", "--off-heap-ram=4096"]); }); + describe("silent logging", () => { + it("should log command output", async () => { + const queryDir = getOnDiskWorkspaceFolders()[0]; + await cli.resolveQueries(queryDir); + + expect(logSpy).toHaveBeenCalled(); + }); + + it("shouldn't log command output if the `silent` flag is set", async () => { + const queryDir = getOnDiskWorkspaceFolders()[0]; + const silent = true; + await cli.resolveQueries(queryDir, silent); + + expect(logSpy).not.toHaveBeenCalled(); + }); + }); + itWithCodeQL()("should resolve query packs", async () => { const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders()); // Depending on the version of the CLI, the qlpacks may have different names diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts index a50a08c15..949d79580 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts @@ -546,9 +546,7 @@ describe("SkeletonQueryWizard", () => { dateAdded: 123, } as FullDatabaseOptions); - jest - .spyOn(mockDbItem, "error", "get") - .mockReturnValue(asError("database go boom!")); + mockDbItem.error = asError("database go boom!"); const sortedList = await SkeletonQueryWizard.sortDatabaseItemsByDateAdded([ diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts index 8d8bc9aed..993d0bb1d 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts @@ -222,6 +222,7 @@ describe("Variant Analysis Manager", () => { it("should run a remote query that is part of a qlpack", async () => { await doVariantAnalysisTest({ queryPath: "data-remote-qlpack/in-pack.ql", + expectedPackName: "github/remote-query-pack", filesThatExist: ["in-pack.ql", "lib.qll"], filesThatDoNotExist: [], qlxFilesThatExist: ["in-pack.qlx"], @@ -231,6 +232,7 @@ describe("Variant Analysis Manager", () => { it("should run a remote query that is not part of a qlpack", async () => { await doVariantAnalysisTest({ queryPath: "data-remote-no-qlpack/in-pack.ql", + expectedPackName: "codeql-remote/query", filesThatExist: ["in-pack.ql"], filesThatDoNotExist: ["lib.qll", "not-in-pack.ql"], qlxFilesThatExist: ["in-pack.qlx"], @@ -240,6 +242,7 @@ describe("Variant Analysis Manager", () => { it("should run a remote query that is nested inside a qlpack", async () => { await doVariantAnalysisTest({ queryPath: "data-remote-qlpack-nested/subfolder/in-pack.ql", + expectedPackName: "github/remote-query-pack", filesThatExist: ["subfolder/in-pack.ql", "otherfolder/lib.qll"], filesThatDoNotExist: ["subfolder/not-in-pack.ql"], qlxFilesThatExist: ["subfolder/in-pack.qlx"], @@ -256,6 +259,7 @@ describe("Variant Analysis Manager", () => { await cli.setUseExtensionPacks(true); await doVariantAnalysisTest({ queryPath: "data-remote-qlpack-nested/subfolder/in-pack.ql", + expectedPackName: "github/remote-query-pack", filesThatExist: [ "subfolder/in-pack.ql", "otherfolder/lib.qll", @@ -273,12 +277,14 @@ describe("Variant Analysis Manager", () => { async function doVariantAnalysisTest({ queryPath, + expectedPackName, filesThatExist, qlxFilesThatExist, filesThatDoNotExist, dependenciesToCheck = ["codeql/javascript-all"], }: { queryPath: string; + expectedPackName: string; filesThatExist: string[]; qlxFilesThatExist: string[]; filesThatDoNotExist: string[]; @@ -332,7 +338,7 @@ describe("Variant Analysis Manager", () => { const qlpackContents = load( packFS.fileContents(packFileName).toString("utf-8"), ); - expect(qlpackContents.name).toEqual("codeql-remote/query"); + expect(qlpackContents.name).toEqual(expectedPackName); expect(qlpackContents.version).toEqual("0.0.0"); expect(qlpackContents.dependencies?.["codeql/javascript-all"]).toEqual( "*", diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts index d3e08fdac..8f8c8689f 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts @@ -349,7 +349,12 @@ describe("db panel rendering nodes", () => { expect(item.tooltip).toBeUndefined(); expect(item.iconPath).toBeUndefined(); expect(item.collapsibleState).toBe(TreeItemCollapsibleState.Collapsed); - checkDbItemActions(item, ["canBeSelected", "canBeRenamed", "canBeRemoved"]); + checkDbItemActions(item, [ + "canBeSelected", + "canBeRenamed", + "canBeRemoved", + "canImportCodeSearch", + ]); expect(item.children).toBeTruthy(); expect(item.children.length).toBe(repos.length); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts index 921bf037f..61eeb724f 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts @@ -26,7 +26,7 @@ export function run() { it("should allow ql files to be quick-evaled", async () => { await showQlDocument("query.ql"); - const q = await getQuickEvalContext(undefined); + const q = await getQuickEvalContext(undefined, false); expect( q.quickEvalPosition.fileName.endsWith( join("ql-vscode", "test", "data", "query.ql"), @@ -36,7 +36,7 @@ export function run() { it("should allow qll files to be quick-evaled", async () => { await showQlDocument("library.qll"); - const q = await getQuickEvalContext(undefined); + const q = await getQuickEvalContext(undefined, false); expect( q.quickEvalPosition.fileName.endsWith( join("ql-vscode", "test", "data", "library.qll"), @@ -55,7 +55,7 @@ export function run() { it("should reject non-ql[l] files when running a quick eval", async () => { await showQlDocument("textfile.txt"); - await expect(getQuickEvalContext(undefined)).rejects.toThrow( + await expect(getQuickEvalContext(undefined, false)).rejects.toThrow( "The selected resource is not a CodeQL file", ); }); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts index 034c86f66..a05418c39 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts @@ -9,7 +9,6 @@ import { DatabaseItemImpl, DatabaseManager, DatabaseResolver, - findSourceArchive, FullDatabaseOptions, } from "../../../src/databases/local-databases"; import { Logger } from "../../../src/common"; @@ -32,6 +31,7 @@ import { mockDbOptions, sourceLocationUri, } from "../../factories/databases/databases"; +import { findSourceArchive } from "../../../src/databases/local-databases/database-resolver"; describe("local databases", () => { let databaseManager: DatabaseManager; @@ -327,7 +327,7 @@ describe("local databases", () => { mockDbOptions(), Uri.parse("file:/sourceArchive-uri/"), ); - (db as any)._contents.sourceArchiveUri = undefined; + (db as any).contents.sourceArchiveUri = undefined; expect(() => db.resolveSourceFile("abc")).toThrowError( "Scheme is missing", ); @@ -339,7 +339,7 @@ describe("local databases", () => { mockDbOptions(), Uri.parse("file:/sourceArchive-uri/"), ); - (db as any)._contents.sourceArchiveUri = undefined; + (db as any).contents.sourceArchiveUri = undefined; expect(() => db.resolveSourceFile("http://abc")).toThrowError( "Invalid uri scheme", ); @@ -352,7 +352,7 @@ describe("local databases", () => { mockDbOptions(), Uri.parse("file:/sourceArchive-uri/"), ); - (db as any)._contents.sourceArchiveUri = undefined; + (db as any).contents.sourceArchiveUri = undefined; const resolved = db.resolveSourceFile(undefined); expect(resolved.toString(true)).toBe(dbLocationUri(dir).toString(true)); }); @@ -363,7 +363,7 @@ describe("local databases", () => { mockDbOptions(), Uri.parse("file:/sourceArchive-uri/"), ); - (db as any)._contents.sourceArchiveUri = undefined; + (db as any).contents.sourceArchiveUri = undefined; const resolved = db.resolveSourceFile("file:"); expect(resolved.toString()).toBe("file:///"); }); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts new file mode 100644 index 000000000..563ee6119 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts @@ -0,0 +1,208 @@ +import { + EventEmitter, + FileSystemWatcher, + Uri, + WorkspaceFoldersChangeEvent, + workspace, +} from "vscode"; +import { CodeQLCliServer } from "../../../../src/codeql-cli/cli"; +import { QueryDiscovery } from "../../../../src/queries-panel/query-discovery"; +import { createMockApp } from "../../../__mocks__/appMock"; +import { mockedObject } from "../../utils/mocking.helpers"; +import { basename, join, sep } from "path"; + +describe("QueryDiscovery", () => { + beforeEach(() => { + expect(workspace.workspaceFolders?.length).toEqual(1); + }); + + describe("queries", () => { + it("should return empty list when no QL files are present", async () => { + const resolveQueries = jest.fn().mockResolvedValue([]); + const cli = mockedObject({ + resolveQueries, + }); + + const discovery = new QueryDiscovery(createMockApp({}), cli); + await discovery.refresh(); + const queries = discovery.queries; + + expect(queries).toEqual([]); + expect(resolveQueries).toHaveBeenCalledTimes(1); + }); + + it("should organise query files into directories", async () => { + const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath; + const cli = mockedObject({ + resolveQueries: jest + .fn() + .mockResolvedValue([ + join(workspaceRoot, "dir1/query1.ql"), + join(workspaceRoot, "dir2/query2.ql"), + join(workspaceRoot, "query3.ql"), + ]), + }); + + const discovery = new QueryDiscovery(createMockApp({}), cli); + await discovery.refresh(); + const queries = discovery.queries; + expect(queries).toBeDefined(); + + expect(queries![0].children.length).toEqual(3); + expect(queries![0].children[0].name).toEqual("dir1"); + expect(queries![0].children[0].children.length).toEqual(1); + expect(queries![0].children[0].children[0].name).toEqual("query1.ql"); + expect(queries![0].children[1].name).toEqual("dir2"); + expect(queries![0].children[1].children.length).toEqual(1); + expect(queries![0].children[1].children[0].name).toEqual("query2.ql"); + expect(queries![0].children[2].name).toEqual("query3.ql"); + }); + + it("should collapse directories containing only a single element", async () => { + const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath; + const cli = mockedObject({ + resolveQueries: jest + .fn() + .mockResolvedValue([ + join(workspaceRoot, "dir1/query1.ql"), + join(workspaceRoot, "dir1/dir2/dir3/dir3/query2.ql"), + ]), + }); + + const discovery = new QueryDiscovery(createMockApp({}), cli); + await discovery.refresh(); + const queries = discovery.queries; + expect(queries).toBeDefined(); + + expect(queries![0].children.length).toEqual(1); + expect(queries![0].children[0].name).toEqual("dir1"); + expect(queries![0].children[0].children.length).toEqual(2); + expect(queries![0].children[0].children[0].name).toEqual( + "dir2 / dir3 / dir3", + ); + expect(queries![0].children[0].children[0].children.length).toEqual(1); + expect(queries![0].children[0].children[0].children[0].name).toEqual( + "query2.ql", + ); + expect(queries![0].children[0].children[1].name).toEqual("query1.ql"); + }); + + it("calls resolveQueries once for each workspace folder", async () => { + const workspaceRoots = [ + `${sep}workspace1`, + `${sep}workspace2`, + `${sep}workspace3`, + ]; + jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValueOnce( + workspaceRoots.map((root, index) => ({ + uri: Uri.file(root), + name: basename(root), + index, + })), + ); + + const resolveQueries = jest.fn().mockImplementation((queryDir) => { + const workspaceIndex = workspaceRoots.indexOf(queryDir); + if (workspaceIndex === -1) { + throw new Error("Unexpected workspace"); + } + return Promise.resolve([ + join(queryDir, `query${workspaceIndex + 1}.ql`), + ]); + }); + const cli = mockedObject({ + resolveQueries, + }); + + const discovery = new QueryDiscovery(createMockApp({}), cli); + await discovery.refresh(); + const queries = discovery.queries; + expect(queries).toBeDefined(); + + expect(queries!.length).toEqual(3); + expect(queries![0].children[0].name).toEqual("query1.ql"); + expect(queries![1].children[0].name).toEqual("query2.ql"); + expect(queries![2].children[0].name).toEqual("query3.ql"); + + expect(resolveQueries).toHaveBeenCalledTimes(3); + }); + }); + + describe("onDidChangeQueries", () => { + it("should fire onDidChangeQueries when a watcher fires", async () => { + const onWatcherDidChangeEvent = new EventEmitter(); + const watcher: FileSystemWatcher = { + ignoreCreateEvents: false, + ignoreChangeEvents: false, + ignoreDeleteEvents: false, + onDidCreate: onWatcherDidChangeEvent.event, + onDidChange: onWatcherDidChangeEvent.event, + onDidDelete: onWatcherDidChangeEvent.event, + dispose: () => undefined, + }; + const createFileSystemWatcherSpy = jest.spyOn( + workspace, + "createFileSystemWatcher", + ); + createFileSystemWatcherSpy.mockReturnValue(watcher); + + const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath; + const cli = mockedObject({ + resolveQueries: jest + .fn() + .mockResolvedValue([join(workspaceRoot, "query1.ql")]), + }); + + const discovery = new QueryDiscovery( + createMockApp({ + createEventEmitter: () => new EventEmitter(), + }), + cli, + ); + + const onDidChangeQueriesSpy = jest.fn(); + discovery.onDidChangeQueries(onDidChangeQueriesSpy); + + await discovery.refresh(); + + expect(createFileSystemWatcherSpy).toHaveBeenCalledTimes(2); + expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1); + + onWatcherDidChangeEvent.fire(workspace.workspaceFolders![0].uri); + + await discovery.waitForCurrentRefresh(); + + expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2); + }); + }); + + describe("onDidChangeWorkspaceFolders", () => { + it("should refresh when workspace folders change", async () => { + const onDidChangeWorkspaceFoldersEvent = + new EventEmitter(); + + const discovery = new QueryDiscovery( + createMockApp({ + createEventEmitter: () => new EventEmitter(), + onDidChangeWorkspaceFolders: onDidChangeWorkspaceFoldersEvent.event, + }), + mockedObject({ + resolveQueries: jest.fn().mockResolvedValue([]), + }), + ); + + const onDidChangeQueriesSpy = jest.fn(); + discovery.onDidChangeQueries(onDidChangeQueriesSpy); + + await discovery.refresh(); + + expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1); + + onDidChangeWorkspaceFoldersEvent.fire({ added: [], removed: [] }); + + await discovery.waitForCurrentRefresh(); + + expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts new file mode 100644 index 000000000..080528689 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts @@ -0,0 +1,93 @@ +import { EventEmitter } from "vscode"; +import { + FileTreeDirectory, + FileTreeLeaf, +} from "../../../../src/common/file-tree-nodes"; +import { + QueryDiscoverer, + QueryTreeDataProvider, +} from "../../../../src/queries-panel/query-tree-data-provider"; + +describe("QueryTreeDataProvider", () => { + describe("getChildren", () => { + it("returns no children when queries is undefined", async () => { + const dataProvider = new QueryTreeDataProvider({ + queries: undefined, + onDidChangeQueries: jest.fn(), + }); + + expect(dataProvider.getChildren()).toEqual([]); + }); + + it("returns no children when there are no queries", async () => { + const dataProvider = new QueryTreeDataProvider({ + queries: [], + onDidChangeQueries: jest.fn(), + }); + + expect(dataProvider.getChildren()).toEqual([]); + }); + + it("converts FileTreeNode to QueryTreeViewItem", async () => { + const dataProvider = new QueryTreeDataProvider({ + queries: [ + new FileTreeDirectory("dir1", "dir1", [ + new FileTreeDirectory("dir1/dir2", "dir2", [ + new FileTreeLeaf("dir1/dir2/file1", "file1"), + new FileTreeLeaf("dir1/dir2/file1", "file2"), + ]), + ]), + new FileTreeDirectory("dir3", "dir3", [ + new FileTreeLeaf("dir3/file3", "file3"), + ]), + ], + onDidChangeQueries: jest.fn(), + }); + + expect(dataProvider.getChildren().length).toEqual(2); + + expect(dataProvider.getChildren()[0].label).toEqual("dir1"); + expect(dataProvider.getChildren()[0].children.length).toEqual(1); + expect(dataProvider.getChildren()[0].children[0].label).toEqual("dir2"); + expect(dataProvider.getChildren()[0].children[0].children.length).toEqual( + 2, + ); + expect( + dataProvider.getChildren()[0].children[0].children[0].label, + ).toEqual("file1"); + expect( + dataProvider.getChildren()[0].children[0].children[1].label, + ).toEqual("file2"); + + expect(dataProvider.getChildren()[1].label).toEqual("dir3"); + expect(dataProvider.getChildren()[1].children.length).toEqual(1); + expect(dataProvider.getChildren()[1].children[0].label).toEqual("file3"); + }); + }); + + describe("onDidChangeQueries", () => { + it("should update tree when the queries change", async () => { + const onDidChangeQueriesEmitter = new EventEmitter(); + const queryDiscoverer: QueryDiscoverer = { + queries: [ + new FileTreeDirectory("dir1", "dir1", [ + new FileTreeLeaf("dir1/file1", "file1"), + ]), + ], + onDidChangeQueries: onDidChangeQueriesEmitter.event, + }; + + const dataProvider = new QueryTreeDataProvider(queryDiscoverer); + expect(dataProvider.getChildren().length).toEqual(1); + + queryDiscoverer.queries?.push( + new FileTreeDirectory("dir2", "dir2", [ + new FileTreeLeaf("dir2/file2", "file2"), + ]), + ); + onDidChangeQueriesEmitter.fire(); + + expect(dataProvider.getChildren().length).toEqual(2); + }); + }); +}); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts index e85b670a2..49edbef9e 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts @@ -52,12 +52,14 @@ describe("qltest-discovery", () => { }); it("should run discovery", async () => { - const result = await (qlTestDiscover as any).discover(); - expect(result.watchPath).toEqualPath(baseDir); - expect(result.testDirectory.path).toEqualPath(baseDir); - expect(result.testDirectory.name).toBe("My tests"); + await qlTestDiscover.refresh(); + const testDirectory = qlTestDiscover.testDirectory; + expect(testDirectory).toBeDefined(); - let children = result.testDirectory.children; + expect(testDirectory!.path).toEqualPath(baseDir); + expect(testDirectory!.name).toBe("My tests"); + + let children = testDirectory!.children; expect(children.length).toBe(1); expect(children[0].path).toEqualPath(cDir); @@ -83,12 +85,14 @@ describe("qltest-discovery", () => { it("should avoid discovery if a folder does not exist", async () => { await fs.remove(baseDir); - const result = await (qlTestDiscover as any).discover(); - expect(result.watchPath).toEqualPath(baseDir); - expect(result.testDirectory.path).toEqualPath(baseDir); - expect(result.testDirectory.name).toBe("My tests"); + await qlTestDiscover.refresh(); + const testDirectory = qlTestDiscover.testDirectory; + expect(testDirectory).toBeDefined(); - expect(result.testDirectory.children.length).toBe(0); + expect(testDirectory!.path).toEqualPath(baseDir); + expect(testDirectory!.name).toBe("My tests"); + + expect(testDirectory!.children.length).toBe(0); }); }); }); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts index 0c595078d..4f3ac600d 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts @@ -286,7 +286,7 @@ describe("Launcher path", () => { const manager = new DistributionManager( { customCodeQlPath: pathToCmd } as any, {} as any, - undefined as any, + {} as any, ); const result = await manager.getCodeQlPathWithoutVersionCheck(); @@ -304,7 +304,7 @@ describe("Launcher path", () => { const manager = new DistributionManager( { customCodeQlPath: pathToCmd } as any, {} as any, - undefined as any, + {} as any, ); const result = await manager.getCodeQlPathWithoutVersionCheck(); @@ -319,7 +319,7 @@ describe("Launcher path", () => { const manager = new DistributionManager( { customCodeQlPath: pathToCmd } as any, {} as any, - undefined as any, + {} as any, ); const result = await manager.getCodeQlPathWithoutVersionCheck(); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index a4b5ec0e0..a470b2180 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -10,7 +10,7 @@ import { QueryResultType } from "../../../../src/pure/new-messages"; import { readdir, readFile } from "fs-extra"; import { load } from "js-yaml"; import { dirname, join } from "path"; -import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries/index"; +import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries"; import * as helpers from "../../../../src/helpers"; import { RedactableError } from "../../../../src/pure/errors"; @@ -77,6 +77,7 @@ describe("runQuery", () => { { queryPath: expect.stringMatching(/FetchExternalApis\.ql/), quickEvalPosition: undefined, + quickEvalCountOnly: false, }, false, [], @@ -161,18 +162,20 @@ describe("readQueryResults", () => { name: "#select", rows: 10, columns: [ - { name: "apiName", kind: "s" }, - { name: "supported", kind: "b" }, { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, ], }, { name: "#select2", rows: 10, columns: [ - { name: "apiName", kind: "s" }, - { name: "supported", kind: "b" }, { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, ], }, ], @@ -191,9 +194,10 @@ describe("readQueryResults", () => { name: "#select", rows: 10, columns: [ - { name: "apiName", kind: "s" }, - { name: "supported", kind: "b" }, { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, ], }, ], @@ -201,9 +205,10 @@ describe("readQueryResults", () => { }); const decodedResultSet = { columns: [ - { name: "apiName", kind: "String" }, - { name: "supported", kind: "Boolean" }, - { name: "usage", kind: "Entity" }, + { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, ], tuples: [ [ diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts index 21be77229..d6afc9cf6 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts @@ -1,17 +1,4 @@ -import { - EnvironmentVariableCollection, - EnvironmentVariableMutator, - Event, - ExtensionContext, - ExtensionMode, - Memento, - SecretStorage, - SecretStorageChangeEvent, - Uri, - window, - workspace, - WorkspaceFolder, -} from "vscode"; +import { Uri, window, workspace, WorkspaceFolder } from "vscode"; import { dump } from "js-yaml"; import * as tmp from "tmp"; import { join } from "path"; @@ -28,7 +15,6 @@ import { DirResult } from "tmp"; import { getFirstWorkspaceFolder, getInitialQueryContents, - InvocationRateLimiter, isFolderAlreadyInWorkspace, isLikelyDatabaseRoot, isLikelyDbLanguageFolder, @@ -45,118 +31,6 @@ import { Setting } from "../../../src/config"; import { createMockCommandManager } from "../../__mocks__/commandsMock"; describe("helpers", () => { - describe("Invocation rate limiter", () => { - // 1 January 2020 - let currentUnixTime = 1577836800; - - function createDate(dateString?: string): Date { - if (dateString) { - return new Date(dateString); - } - const numMillisecondsPerSecond = 1000; - return new Date(currentUnixTime * numMillisecondsPerSecond); - } - - function createInvocationRateLimiter( - funcIdentifier: string, - func: () => Promise, - ): InvocationRateLimiter { - return new InvocationRateLimiter( - new MockExtensionContext(), - funcIdentifier, - func, - (s) => createDate(s), - ); - } - - it("initially invokes function", async () => { - let numTimesFuncCalled = 0; - const invocationRateLimiter = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncCalled++; - }, - ); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); - expect(numTimesFuncCalled).toBe(1); - }); - - it("doesn't invoke function again if no time has passed", async () => { - let numTimesFuncCalled = 0; - const invocationRateLimiter = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncCalled++; - }, - ); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); - expect(numTimesFuncCalled).toBe(1); - }); - - it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => { - let numTimesFuncCalled = 0; - const invocationRateLimiter = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncCalled++; - }, - ); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100); - currentUnixTime += 1; - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2); - expect(numTimesFuncCalled).toBe(1); - }); - - it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => { - let numTimesFuncCalled = 0; - const invocationRateLimiter = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncCalled++; - }, - ); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0); - expect(numTimesFuncCalled).toBe(2); - }); - - it("invokes function again after requested time since last invocation has elapsed", async () => { - let numTimesFuncCalled = 0; - const invocationRateLimiter = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncCalled++; - }, - ); - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1); - currentUnixTime += 1; - await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1); - expect(numTimesFuncCalled).toBe(2); - }); - - it("invokes functions with different rate limiters", async () => { - let numTimesFuncACalled = 0; - const invocationRateLimiterA = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncACalled++; - }, - ); - let numTimesFuncBCalled = 0; - const invocationRateLimiterB = createInvocationRateLimiter( - "funcid", - async () => { - numTimesFuncBCalled++; - }, - ); - await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100); - await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100); - expect(numTimesFuncACalled).toBe(1); - expect(numTimesFuncBCalled).toBe(1); - }); - }); - describe("codeql-database.yml tests", () => { let dir: tmp.DirResult; let language: QueryLanguage; @@ -250,116 +124,6 @@ describe("helpers", () => { }); }); - class MockExtensionContext implements ExtensionContext { - extensionMode: ExtensionMode = 3; - subscriptions: Array<{ dispose(): unknown }> = []; - workspaceState: Memento = new MockMemento(); - globalState = new MockGlobalStorage(); - extensionPath = ""; - asAbsolutePath(_relativePath: string): string { - throw new Error("Method not implemented."); - } - storagePath = ""; - globalStoragePath = ""; - logPath = ""; - extensionUri = Uri.parse(""); - environmentVariableCollection = new MockEnvironmentVariableCollection(); - secrets = new MockSecretStorage(); - storageUri = Uri.parse(""); - globalStorageUri = Uri.parse(""); - logUri = Uri.parse(""); - extension: any; - } - - class MockEnvironmentVariableCollection - implements EnvironmentVariableCollection - { - [Symbol.iterator](): Iterator< - [variable: string, mutator: EnvironmentVariableMutator], - any, - undefined - > { - throw new Error("Method not implemented."); - } - persistent = false; - replace(_variable: string, _value: string): void { - throw new Error("Method not implemented."); - } - append(_variable: string, _value: string): void { - throw new Error("Method not implemented."); - } - prepend(_variable: string, _value: string): void { - throw new Error("Method not implemented."); - } - get(_variable: string): EnvironmentVariableMutator | undefined { - throw new Error("Method not implemented."); - } - forEach( - _callback: ( - variable: string, - mutator: EnvironmentVariableMutator, - collection: EnvironmentVariableCollection, - ) => any, - _thisArg?: any, - ): void { - throw new Error("Method not implemented."); - } - delete(_variable: string): void { - throw new Error("Method not implemented."); - } - clear(): void { - throw new Error("Method not implemented."); - } - } - - class MockMemento implements Memento { - keys(): readonly string[] { - throw new Error("Method not implemented."); - } - map = new Map(); - - /** - * Return a value. - * - * @param key A string. - * @param defaultValue A value that should be returned when there is no - * value (`undefined`) with the given key. - * @return The stored value or the defaultValue. - */ - get(key: string, defaultValue?: T): T { - return this.map.has(key) ? this.map.get(key) : defaultValue; - } - - /** - * Store a value. The value must be JSON-stringifyable. - * - * @param key A string. - * @param value A value. MUST not contain cyclic references. - */ - async update(key: string, value: any): Promise { - this.map.set(key, value); - } - } - - class MockGlobalStorage extends MockMemento { - public setKeysForSync(_keys: string[]): void { - return; - } - } - - class MockSecretStorage implements SecretStorage { - get(_key: string): Thenable { - throw new Error("Method not implemented."); - } - store(_key: string, _value: string): Thenable { - throw new Error("Method not implemented."); - } - delete(_key: string): Thenable { - throw new Error("Method not implemented."); - } - onDidChange!: Event; - } - it("should report stream progress", () => { const progressSpy = jest.fn(); const mockReadable = { diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts index 1e7108abf..fe9a51c22 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts @@ -225,6 +225,7 @@ describe("query-results", () => { resultsPath, interpretedResultsPath, sourceInfo, + undefined, ); }, 2 * 60 * 1000, // up to 2 minutes per test @@ -249,6 +250,7 @@ describe("query-results", () => { resultsPath, interpretedResultsPath, sourceInfo, + undefined, ); }, 2 * 60 * 1000, // up to 2 minutes per test diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts index 628355fb7..f4d9a3902 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts @@ -40,17 +40,11 @@ describe("test-runner", () => { Uri.file("/path/to/test/dir/dir.testproj"), undefined, mockedObject({ displayName: "custom display name" }), - (_) => { - /* no change event listener */ - }, ); const postTestDatabaseItem = new DatabaseItemImpl( Uri.file("/path/to/test/dir/dir.testproj"), undefined, mockedObject({ displayName: "default name" }), - (_) => { - /* no change event listener */ - }, ); beforeEach(() => {