mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into robertbrignull/database-prompting

Browse Source
This commit is contained in:
Robert
2024-03-28 13:12:00 +00:00
parent ff889b73e1 7a41d9c605
commit 3cae3ccc3c
106 changed files with 4782 additions and 6507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/codeql/queries/ProgressBar.qll vendored Normal file
View File

@@ -0,0 +1,16 @@
import javascript
class WithProgressCall extends CallExpr {
WithProgressCall() { this.getCalleeName() = "withProgress" }
predicate usesToken() { exists(this.getTokenParameter()) }
Parameter getTokenParameter() { result = this.getArgument(0).(Function).getParameter(1) }
Property getCancellableProperty() { result = this.getArgument(1).(ObjectExpr).getPropertyByName("cancellable") }
predicate isCancellable() {
this.getCancellableProperty().getInit().(BooleanLiteral).getBoolValue() =
true
}
}

20
.github/codeql/queries/progress-not-cancellable.ql vendored Normal file
View File

@@ -0,0 +1,20 @@
/**
* @name Using token for non-cancellable progress bar
* @kind problem
* @problem.severity warning
* @id vscode-codeql/progress-not-cancellable
* @description If we call `withProgress` without `cancellable: true` then the
* token that is given to us should be ignored because it won't ever be cancelled.
* This makes the code more confusing as it tries to account for cases that can't
* happen. The fix is to either not use the token or make the progress bar cancellable.
*/
import javascript
import ProgressBar
from WithProgressCall t
where not t.isCancellable() and t.usesToken()
select t,
"The $@ should not be used when the progress bar is not cancellable. Either stop using the $@ or mark the progress bar as cancellable.",
t.getTokenParameter(), t.getTokenParameter().getName(), t.getTokenParameter(),
t.getTokenParameter().getName()

18
.github/codeql/queries/token-not-used.ql vendored Normal file
View File

@@ -0,0 +1,18 @@
/**
* @name Don't ignore the token for a cancellable progress bar
* @kind problem
* @problem.severity warning
* @id vscode-codeql/token-not-used
* @description If we call `withProgress` with `cancellable: true` but then
* ignore the token that is given to us, it will lead to a poor user experience
* because the progress bar will appear to be canceled but it will not actually
* affect the background process. Either check the token and respect when it
* has been cancelled, or mark the progress bar as not cancellable.
*/
import javascript
import ProgressBar
from WithProgressCall t
where t.isCancellable() and not t.usesToken()
select t, "This progress bar is $@ but the token is not used. Either use the token or mark the progress bar as not cancellable.", t.getCancellableProperty(), "cancellable"

14
.github/dependabot.yml vendored
View File

@@ -13,6 +13,20 @@ updates:
# are unrelated to the Node version, so we allow those.
- dependency-name: "@types/node"
update-types: ["version-update:semver-major", "version-update:semver-minor"]
groups:
octokit:
patterns:
- "@octokit/*"
storybook:
patterns:
- "@storybook/*"
- "storybook"
testing-library:
patterns:
- "@testing-library/*"
typescript-eslint:
patterns:
- "@typescript-eslint/*"
- package-ecosystem: "github-actions"
directory: "/"
schedule:

16
extensions/ql-vscode/.babelrc.json
View File

@@ -1,16 +0,0 @@
{
"sourceType": "unambiguous",
"presets": [
[
"@babel/preset-env",
{
"targets": {
"chrome": 100
}
}
],
"@babel/preset-typescript",
"@babel/preset-react"
],
"plugins": []
}

6
extensions/ql-vscode/.storybook/main.ts
View File

@@ -1,7 +1,7 @@
import type { StorybookConfig } from "@storybook/react-webpack5";
import type { StorybookConfig } from "@storybook/react-vite";
const config: StorybookConfig = {
stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
stories: ["../src/**/*.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
addons: [
"@storybook/addon-links",
"@storybook/addon-essentials",
@@ -10,7 +10,7 @@ const config: StorybookConfig = {
"./vscode-theme-addon/preset.ts",
],
framework: {
name: "@storybook/react-webpack5",
name: "@storybook/react-vite",
options: {},
},
docs: {

4
extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx
View File

@@ -5,10 +5,10 @@ import { useCallback } from "react";
import { useGlobals } from "@storybook/manager-api";
import {
IconButton,
Icons,
TooltipLinkList,
WithTooltip,
} from "@storybook/components";
import { DashboardIcon } from "@storybook/icons";
import { themeNames, VSCodeTheme } from "./theme";
@@ -53,7 +53,7 @@ export const ThemeSelector: FunctionComponent = () => {
title="Change the theme of the preview"
active={vscodeTheme !== VSCodeTheme.Dark}
>
<Icons icon="dashboard" />
<DashboardIcon />
</IconButton>
</WithTooltip>
);

39
extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts
View File

@@ -1,3 +1,5 @@
/// <reference types="vite/client" />
import { useEffect } from "react";
import type {
PartialStoryFn as StoryFunction,
@@ -6,31 +8,20 @@ import type {
import { VSCodeTheme } from "./theme";
import darkThemeStyle from "../../src/stories/vscode-theme-dark.css?url";
import lightThemeStyle from "../../src/stories/vscode-theme-light.css?url";
import lightHighContrastThemeStyle from "../../src/stories/vscode-theme-light-high-contrast.css?url";
import darkHighContrastThemeStyle from "../../src/stories/vscode-theme-dark-high-contrast.css?url";
import githubLightDefaultThemeStyle from "../../src/stories/vscode-theme-github-light-default.css?url";
import githubDarkDefaultThemeStyle from "../../src/stories/vscode-theme-github-dark-default.css?url";
const themeFiles: { [key in VSCodeTheme]: string } = {
[VSCodeTheme.Dark]:
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
require("!file-loader?modules!../../src/stories/vscode-theme-dark.css")
.default,
[VSCodeTheme.Light]:
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
require("!file-loader?modules!../../src/stories/vscode-theme-light.css")
.default,
[VSCodeTheme.LightHighContrast]:
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
require("!file-loader?modules!../../src/stories/vscode-theme-light-high-contrast.css")
.default,
[VSCodeTheme.DarkHighContrast]:
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
require("!file-loader?modules!../../src/stories/vscode-theme-dark-high-contrast.css")
.default,
[VSCodeTheme.GitHubLightDefault]:
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
require("!file-loader?modules!../../src/stories/vscode-theme-github-light-default.css")
.default,
[VSCodeTheme.GitHubDarkDefault]:
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
require("!file-loader?modules!../../src/stories/vscode-theme-github-dark-default.css")
.default,
[VSCodeTheme.Dark]: darkThemeStyle,
[VSCodeTheme.Light]: lightThemeStyle,
[VSCodeTheme.LightHighContrast]: lightHighContrastThemeStyle,
[VSCodeTheme.DarkHighContrast]: darkHighContrastThemeStyle,
[VSCodeTheme.GitHubLightDefault]: githubLightDefaultThemeStyle,
[VSCodeTheme.GitHubDarkDefault]: githubDarkDefaultThemeStyle,
};
export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {

7
extensions/ql-vscode/CHANGELOG.md
View File

@@ -2,6 +2,13 @@
## [UNRELEASED]
- Add new supported source and sink kinds in the CodeQL Model Editor [#3511](https://github.com/github/vscode-codeql/pull/3511)
## 1.12.4 - 20 March 2024
- Don't show notification after local query cancellation. [#3489](https://github.com/github/vscode-codeql/pull/3489)
- Databases created from [CodeQL test cases](https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/testing-custom-queries) are now copied into a shared VS Code storage location. This avoids a bug where re-running test cases would fail if the test's database is already imported into the workspace. [#3433](https://github.com/github/vscode-codeql/pull/3433)
## 1.12.3 - 29 February 2024
- Update variant analysis view to show when cancelation is in progress. [#3405](https://github.com/github/vscode-codeql/pull/3405)

7174
extensions/ql-vscode/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

61
extensions/ql-vscode/package.json
View File

@@ -4,7 +4,7 @@
"description": "CodeQL for Visual Studio Code",
"author": "GitHub",
"private": true,
"version": "1.12.4",
"version": "1.12.5",
"publisher": "GitHub",
"license": "MIT",
"icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -738,6 +738,10 @@
"command": "codeQL.setCurrentDatabase",
"title": "CodeQL: Set Current Database"
},
{
"command": "codeQL.importTestDatabase",
"title": "CodeQL: (Re-)Import Test Database"
},
{
"command": "codeQL.getCurrentDatabase",
"title": "CodeQL: Get Current Database"
@@ -1322,7 +1326,12 @@
{
"command": "codeQL.setCurrentDatabase",
"group": "9_qlCommands",
"when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
"when": "resourceExtname != .testproj && (resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zipz)"
},
{
"command": "codeQL.importTestDatabase",
"group": "9_qlCommands",
"when": "explorerResourceIsFolder && resourceExtname == .testproj"
},
{
"command": "codeQL.viewAstContextExplorer",
@@ -1476,6 +1485,10 @@
"command": "codeQL.setCurrentDatabase",
"when": "false"
},
{
"command": "codeQL.importTestDatabase",
"when": "false"
},
{
"command": "codeQL.getCurrentDatabase",
"when": "false"
@@ -1943,7 +1956,7 @@
"@octokit/rest": "^20.0.2",
"@vscode/codicons": "^0.0.35",
"@vscode/debugadapter": "^1.59.0",
"@vscode/debugprotocol": "^1.59.0",
"@vscode/debugprotocol": "^1.65.0",
"@vscode/webview-ui-toolkit": "^1.0.1",
"ajv": "^8.11.0",
"child-process-promise": "^2.2.1",
@@ -1975,27 +1988,29 @@
"devDependencies": {
"@babel/core": "^7.24.0",
"@babel/plugin-transform-modules-commonjs": "^7.18.6",
"@babel/preset-env": "^7.23.9",
"@babel/preset-env": "^7.24.0",
"@babel/preset-react": "^7.18.6",
"@babel/preset-typescript": "^7.21.4",
"@faker-js/faker": "^8.4.1",
"@github/markdownlint-github": "^0.6.0",
"@octokit/plugin-throttling": "^8.0.0",
"@playwright/test": "^1.40.1",
"@storybook/addon-a11y": "^7.6.15",
"@storybook/addon-actions": "^7.1.0",
"@storybook/addon-essentials": "^7.1.0",
"@storybook/addon-interactions": "^7.1.0",
"@storybook/addon-links": "^7.1.0",
"@storybook/components": "^7.6.17",
"@storybook/csf": "^0.1.1",
"@storybook/manager-api": "^7.6.7",
"@storybook/react": "^7.1.0",
"@storybook/react-webpack5": "^7.6.12",
"@storybook/theming": "^7.6.12",
"@storybook/addon-a11y": "^8.0.2",
"@storybook/addon-actions": "^8.0.2",
"@storybook/addon-essentials": "^8.0.2",
"@storybook/addon-interactions": "^8.0.2",
"@storybook/addon-links": "^8.0.2",
"@storybook/blocks": "^8.0.2",
"@storybook/components": "^8.0.2",
"@storybook/csf": "^0.1.3",
"@storybook/icons": "^1.2.9",
"@storybook/manager-api": "^8.0.2",
"@storybook/react": "^8.0.2",
"@storybook/react-vite": "^8.0.2",
"@storybook/theming": "^8.0.2",
"@testing-library/dom": "^9.3.4",
"@testing-library/jest-dom": "^6.4.2",
"@testing-library/react": "^14.2.1",
"@testing-library/react": "^14.2.2",
"@testing-library/user-event": "^14.5.2",
"@types/child-process-promise": "^2.2.1",
"@types/d3": "^7.4.0",
@@ -2004,7 +2019,7 @@
"@types/fs-extra": "^11.0.1",
"@types/gulp": "^4.0.9",
"@types/gulp-replace": "^1.1.0",
"@types/jest": "^29.0.2",
"@types/jest": "^29.5.12",
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "18.17.*",
@@ -2018,10 +2033,9 @@
"@types/tar-stream": "^3.1.3",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.2.6",
"@types/unzipper": "^0.10.1",
"@types/vscode": "^1.82.0",
"@types/yauzl": "^2.10.3",
"@typescript-eslint/eslint-plugin": "^6.19.0",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"@vscode/test-electron": "^2.3.9",
"@vscode/vsce": "^2.24.0",
@@ -2029,7 +2043,6 @@
"applicationinsights": "^2.9.4",
"cosmiconfig": "^9.0.0",
"cross-env": "^7.0.3",
"css-loader": "^6.10.0",
"del": "^6.0.0",
"eslint": "^8.56.0",
"eslint-config-prettier": "^9.0.0",
@@ -2043,7 +2056,6 @@
"eslint-plugin-react": "^7.31.8",
"eslint-plugin-react-hooks": "^4.6.0",
"eslint-plugin-storybook": "^0.8.0",
"file-loader": "^6.2.0",
"glob": "^10.0.0",
"gulp": "^4.0.2",
"gulp-esbuild": "^0.12.0",
@@ -2056,19 +2068,18 @@
"lint-staged": "^15.0.2",
"markdownlint-cli2": "^0.12.1",
"markdownlint-cli2-formatter-pretty": "^0.0.5",
"mini-css-extract-plugin": "^2.8.1",
"npm-run-all": "^4.1.5",
"patch-package": "^8.0.0",
"prettier": "^3.2.5",
"storybook": "^7.6.15",
"storybook": "^8.0.2",
"tar-stream": "^3.1.7",
"through2": "^4.0.2",
"ts-jest": "^29.0.1",
"ts-json-schema-generator": "^1.1.2",
"ts-loader": "^9.4.2",
"ts-node": "^10.7.0",
"ts-unused-exports": "^10.0.0",
"typescript": "^5.0.2"
"typescript": "^5.0.2",
"vite": "^5.2.0"
},
"lint-staged": {
"./**/*.{json,css,scss}": [

453
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -36,6 +36,7 @@ import type { Position } from "../query-server/messages";
import { LOGGING_FLAGS } from "./cli-command";
import type { CliFeatures, VersionAndFeatures } from "./cli-version";
import { ExitCodeError, getCliError } from "./cli-errors";
import { UserCancellationException } from "../common/vscode/progress";
/**
* The version of the SARIF format that we are using.
@@ -217,6 +218,37 @@ type VersionChangedListener = (
newVersionAndFeatures: VersionAndFeatures | undefined,
) => void;
type RunOptions = {
/**
* Used to output progress messages, e.g. to the status bar.
*/
progressReporter?: ProgressReporter;
/**
* Used for responding to interactive output on stdout/stdin.
*/
onLine?: OnLineCallback;
/**
* If true, don't print logs to the CodeQL extension log.
*/
silent?: boolean;
/**
* If true, run this command in a new process rather than in the CLI server.
*/
runInNewProcess?: boolean;
/**
* If runInNewProcess is true, allows cancelling the command. If runInNewProcess
* is false or not specified, this option is ignored.
*/
token?: CancellationToken;
};
type JsonRunOptions = RunOptions & {
/**
* Whether to add commandline arguments to specify the format as JSON.
*/
addFormat?: boolean;
};
/**
* This class manages a cli server started by `codeql execute cli-server` to
* run commands without the overhead of starting a new java
@@ -369,9 +401,6 @@ export class CodeQLCliServer implements Disposable {
onLine?: OnLineCallback,
silent?: boolean,
): Promise<string> {
const stderrBuffers: Buffer[] = [];
// The current buffer of stderr of a single line. To be used for logging.
let currentLineStderrBuffer: Buffer = Buffer.alloc(0);
if (this.commandInProcess) {
throw new Error("runCodeQlCliInternal called while cli was running");
}
@@ -383,8 +412,6 @@ export class CodeQLCliServer implements Disposable {
}
// Grab the process so that typescript know that it is always defined.
const process = this.process;
// The array of fragments of stdout
const stdoutBuffers: Buffer[] = [];
// Compute the full args array
const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
@@ -396,26 +423,160 @@ export class CodeQLCliServer implements Disposable {
);
}
try {
await new Promise<void>((resolve, reject) => {
// Start listening to stdout
process.stdout.addListener("data", (newData: Buffer) => {
if (onLine) {
void (async () => {
const response = await onLine(newData.toString("utf-8"));
return await this.handleProcessOutput(process, {
handleNullTerminator: true,
onListenStart: (process) => {
// Write the command followed by a null terminator.
process.stdin.write(JSON.stringify(args), "utf8");
process.stdin.write(this.nullBuffer);
},
description,
args,
silent,
onLine,
});
} catch (err) {
// Kill the process if it isn't already dead.
this.killProcessIfRunning();
if (!response) {
return;
}
throw err;
}
} finally {
this.commandInProcess = false;
// start running the next command immediately
this.runNext();
}
}
process.stdin.write(`${response}${EOL}`);
private async runCodeQlCliInNewProcess(
command: string[],
commandArgs: string[],
description: string,
onLine?: OnLineCallback,
silent?: boolean,
token?: CancellationToken,
): Promise<string> {
const codeqlPath = await this.getCodeQlPath();
// Remove newData from stdoutBuffers because the data has been consumed
// by the onLine callback.
stdoutBuffers.splice(stdoutBuffers.indexOf(newData), 1);
})();
}
const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
const argsString = args.join(" ");
stdoutBuffers.push(newData);
// If we are running silently, we don't want to print anything to the console.
if (!silent) {
void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
}
const abortController = new AbortController();
const process = spawnChildProcess(codeqlPath, args, {
signal: abortController.signal,
});
if (!process || !process.pid) {
throw new Error(
`Failed to start ${description} using command ${codeqlPath} ${argsString}.`,
);
}
// We need to ensure that we're not killing the same process twice (since this may kill
// another process with the same PID), so keep track of whether we've already exited.
let exited = false;
process.on("exit", () => {
exited = true;
});
const cancellationRegistration = token?.onCancellationRequested((_e) => {
abortController.abort("Token was cancelled.");
if (process.pid && !exited) {
tk(process.pid);
}
});
try {
return await this.handleProcessOutput(process, {
handleNullTerminator: false,
description,
args,
silent,
onLine,
});
} catch (e) {
// If cancellation was requested, the error is probably just because the process was exited with SIGTERM.
if (token?.isCancellationRequested) {
void this.logger.log(
`The process was cancelled and exited with: ${getErrorMessage(e)}`,
);
throw new UserCancellationException(
`Command ${argsString} was cancelled.`,
true, // Don't show a warning message when the user manually cancelled the command.
);
}
throw e;
} finally {
process.stdin.end();
if (!exited) {
tk(process.pid);
}
process.stdout.destroy();
process.stderr.destroy();
cancellationRegistration?.dispose();
}
}
private async handleProcessOutput(
process: ChildProcessWithoutNullStreams,
{
handleNullTerminator,
args,
description,
onLine,
onListenStart,
silent,
}: {
handleNullTerminator: boolean;
args: string[];
description: string;
onLine?: OnLineCallback;
onListenStart?: (process: ChildProcessWithoutNullStreams) => void;
silent?: boolean;
},
): Promise<string> {
const stderrBuffers: Buffer[] = [];
// The current buffer of stderr of a single line. To be used for logging.
let currentLineStderrBuffer: Buffer = Buffer.alloc(0);
// The listeners of the process. Declared here so they can be removed in the finally block.
let stdoutListener: ((newData: Buffer) => void) | undefined = undefined;
let stderrListener: ((newData: Buffer) => void) | undefined = undefined;
let closeListener: ((code: number | null) => void) | undefined = undefined;
let errorListener: ((err: Error) => void) | undefined = undefined;
try {
// The array of fragments of stdout
const stdoutBuffers: Buffer[] = [];
await new Promise<void>((resolve, reject) => {
stdoutListener = (newData: Buffer) => {
if (onLine) {
void (async () => {
const response = await onLine(newData.toString("utf-8"));
if (!response) {
return;
}
process.stdin.write(`${response}${EOL}`);
// Remove newData from stdoutBuffers because the data has been consumed
// by the onLine callback.
stdoutBuffers.splice(stdoutBuffers.indexOf(newData), 1);
})();
}
stdoutBuffers.push(newData);
if (handleNullTerminator) {
// If the buffer ends in '0' then exit.
// We don't have to check the middle as no output will be written after the null until
// the next command starts
@@ -425,89 +586,112 @@ export class CodeQLCliServer implements Disposable {
) {
resolve();
}
});
// Listen to stderr
process.stderr.addListener("data", (newData: Buffer) => {
stderrBuffers.push(newData);
}
};
stderrListener = (newData: Buffer) => {
stderrBuffers.push(newData);
if (!silent) {
currentLineStderrBuffer = Buffer.concat([
currentLineStderrBuffer,
newData,
]);
if (!silent) {
currentLineStderrBuffer = Buffer.concat([
currentLineStderrBuffer,
newData,
]);
// Print the stderr to the logger as it comes in. We need to ensure that
// we don't split messages on the same line, so we buffer the stderr and
// split it on EOLs.
const eolBuffer = Buffer.from(EOL);
// Print the stderr to the logger as it comes in. We need to ensure that
// we don't split messages on the same line, so we buffer the stderr and
// split it on EOLs.
const eolBuffer = Buffer.from(EOL);
let hasCreatedSubarray = false;
let hasCreatedSubarray = false;
let eolIndex;
while (
(eolIndex = currentLineStderrBuffer.indexOf(eolBuffer)) !== -1
) {
const line = currentLineStderrBuffer.subarray(0, eolIndex);
void this.logger.log(line.toString("utf-8"));
currentLineStderrBuffer = currentLineStderrBuffer.subarray(
eolIndex + eolBuffer.length,
);
hasCreatedSubarray = true;
}
// We have created a subarray, which means that the complete original buffer is now referenced
// by the subarray. We need to create a new buffer to avoid memory leaks.
if (hasCreatedSubarray) {
currentLineStderrBuffer = Buffer.from(currentLineStderrBuffer);
}
let eolIndex;
while (
(eolIndex = currentLineStderrBuffer.indexOf(eolBuffer)) !== -1
) {
const line = currentLineStderrBuffer.subarray(0, eolIndex);
void this.logger.log(line.toString("utf-8"));
currentLineStderrBuffer = currentLineStderrBuffer.subarray(
eolIndex + eolBuffer.length,
);
hasCreatedSubarray = true;
}
});
// Listen for process exit.
process.addListener("close", (code) =>
reject(new ExitCodeError(code)),
);
// Write the command followed by a null terminator.
process.stdin.write(JSON.stringify(args), "utf8");
process.stdin.write(this.nullBuffer);
});
// Join all the data together
const fullBuffer = Buffer.concat(stdoutBuffers);
// Make sure we remove the terminator;
const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
if (!silent) {
void this.logger.log(currentLineStderrBuffer.toString("utf8"));
currentLineStderrBuffer = Buffer.alloc(0);
void this.logger.log("CLI command succeeded.");
}
return data;
} catch (err) {
// Kill the process if it isn't already dead.
this.killProcessIfRunning();
// Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
const cliError = getCliError(
err,
stderrBuffers.length > 0
? Buffer.concat(stderrBuffers).toString("utf8")
: undefined,
description,
args,
);
cliError.stack += getErrorStack(err);
throw cliError;
} finally {
if (!silent && currentLineStderrBuffer.length > 0) {
void this.logger.log(currentLineStderrBuffer.toString("utf8"));
}
// Remove the listeners we set up.
process.stdout.removeAllListeners("data");
process.stderr.removeAllListeners("data");
process.removeAllListeners("close");
// We have created a subarray, which means that the complete original buffer is now referenced
// by the subarray. We need to create a new buffer to avoid memory leaks.
if (hasCreatedSubarray) {
currentLineStderrBuffer = Buffer.from(currentLineStderrBuffer);
}
}
};
closeListener = (code) => {
if (handleNullTerminator) {
reject(new ExitCodeError(code));
} else {
if (code === 0) {
resolve();
} else {
reject(new ExitCodeError(code));
}
}
};
errorListener = (err) => {
reject(err);
};
// Start listening to stdout
process.stdout.addListener("data", stdoutListener);
// Listen to stderr
process.stderr.addListener("data", stderrListener);
// Listen for process exit.
process.addListener("close", closeListener);
// Listen for errors
process.addListener("error", errorListener);
onListenStart?.(process);
});
// Join all the data together
const fullBuffer = Buffer.concat(stdoutBuffers);
// Make sure we remove the terminator
const data = fullBuffer.toString(
"utf8",
0,
handleNullTerminator ? fullBuffer.length - 1 : fullBuffer.length,
);
if (!silent) {
void this.logger.log(currentLineStderrBuffer.toString("utf8"));
currentLineStderrBuffer = Buffer.alloc(0);
void this.logger.log("CLI command succeeded.");
}
return data;
} catch (err) {
// Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
const cliError = getCliError(
err,
stderrBuffers.length > 0
? Buffer.concat(stderrBuffers).toString("utf8")
: undefined,
description,
args,
);
cliError.stack += getErrorStack(err);
throw cliError;
} finally {
this.commandInProcess = false;
// start running the next command immediately
this.runNext();
if (!silent && currentLineStderrBuffer.length > 0) {
void this.logger.log(currentLineStderrBuffer.toString("utf8"));
}
// Remove the listeners we set up.
if (stdoutListener) {
process.stdout.removeListener("data", stdoutListener);
}
if (stderrListener) {
process.stderr.removeListener("data", stderrListener);
}
if (closeListener) {
process.removeListener("close", closeListener);
}
if (errorListener) {
process.removeListener("error", errorListener);
}
}
}
@@ -624,6 +808,10 @@ export class CodeQLCliServer implements Disposable {
* @param description Description of the action being run, to be shown in log and error messages.
* @param progressReporter Used to output progress messages, e.g. to the status bar.
* @param onLine Used for responding to interactive output on stdout/stdin.
* @param silent If true, don't print logs to the CodeQL extension log.
* @param runInNewProcess If true, run this command in a new process rather than in the CLI server.
* @param token If runInNewProcess is true, allows cancelling the command. If runInNewProcess
* is false or not specified, this option is ignored.
* @returns The contents of the command's stdout, if the command succeeded.
*/
runCodeQlCliCommand(
@@ -634,16 +822,25 @@ export class CodeQLCliServer implements Disposable {
progressReporter,
onLine,
silent = false,
}: {
progressReporter?: ProgressReporter;
onLine?: OnLineCallback;
silent?: boolean;
} = {},
runInNewProcess = false,
token,
}: RunOptions = {},
): Promise<string> {
if (progressReporter) {
progressReporter.report({ message: description });
}
if (runInNewProcess) {
return this.runCodeQlCliInNewProcess(
command,
commandArgs,
description,
onLine,
silent,
token,
);
}
return new Promise((resolve, reject) => {
// Construct the command that actually does the work
const callback = (): void => {
@@ -676,24 +873,13 @@ export class CodeQLCliServer implements Disposable {
* @param description Description of the action being run, to be shown in log and error messages.
* @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
* @param progressReporter Used to output progress messages, e.g. to the status bar.
* @param onLine Used for responding to interactive output on stdout/stdin.
* @returns The contents of the command's stdout, if the command succeeded.
*/
async runJsonCodeQlCliCommand<OutputType>(
command: string[],
commandArgs: string[],
description: string,
{
addFormat = true,
progressReporter,
onLine,
silent = false,
}: {
addFormat?: boolean;
progressReporter?: ProgressReporter;
onLine?: OnLineCallback;
silent?: boolean;
} = {},
{ addFormat = true, ...runOptions }: JsonRunOptions = {},
): Promise<OutputType> {
let args: string[] = [];
if (addFormat) {
@@ -701,11 +887,12 @@ export class CodeQLCliServer implements Disposable {
args = args.concat(["--format", "json"]);
}
args = args.concat(commandArgs);
const result = await this.runCodeQlCliCommand(command, args, description, {
progressReporter,
onLine,
silent,
});
const result = await this.runCodeQlCliCommand(
command,
args,
description,
runOptions,
);
try {
return JSON.parse(result) as OutputType;
} catch (err) {
@@ -733,21 +920,14 @@ export class CodeQLCliServer implements Disposable {
* @param command The `codeql` command to be run, provided as an array of command/subcommand names.
* @param commandArgs The arguments to pass to the `codeql` command.
* @param description Description of the action being run, to be shown in log and error messages.
* @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
* @param progressReporter Used to output progress messages, e.g. to the status bar.
* @param runOptions Options for running the command.
* @returns The contents of the command's stdout, if the command succeeded.
*/
async runJsonCodeQlCliCommandWithAuthentication<OutputType>(
command: string[],
commandArgs: string[],
description: string,
{
addFormat,
progressReporter,
}: {
addFormat?: boolean;
progressReporter?: ProgressReporter;
} = {},
runOptions: Omit<JsonRunOptions, "onLine"> = {},
): Promise<OutputType> {
const accessToken = await this.app.credentials.getExistingAccessToken();
@@ -758,8 +938,7 @@ export class CodeQLCliServer implements Disposable {
[...extraArgs, ...commandArgs],
description,
{
addFormat,
progressReporter,
...runOptions,
onLine: async (line) => {
if (line.startsWith("Enter value for --github-auth-stdin")) {
try {
@@ -1432,12 +1611,20 @@ export class CodeQLCliServer implements Disposable {
/**
* Downloads a specified pack.
* @param packs The `<package-scope/name[@version]>` of the packs to download.
* @param token The cancellation token. If not specified, the command will be run in the CLI server.
*/
async packDownload(packs: string[]): Promise<PackDownloadResult> {
async packDownload(
packs: string[],
token?: CancellationToken,
): Promise<PackDownloadResult> {
return this.runJsonCodeQlCliCommandWithAuthentication(
["pack", "download"],
packs,
"Downloading packs",
{
runInNewProcess: !!token, // Only run in a new process if a token is provided
token,
},
);
}
@@ -1473,6 +1660,7 @@ export class CodeQLCliServer implements Disposable {
* @param outputBundleFile The path to the output bundle file.
* @param outputPackDir The directory to contain the unbundled output pack.
* @param moreOptions Additional options to be passed to `codeql pack bundle`.
* @param token Cancellation token for the operation.
*/
async packBundle(
sourcePackDir: string,
@@ -1480,6 +1668,7 @@ export class CodeQLCliServer implements Disposable {
outputBundleFile: string,
outputPackDir: string,
moreOptions: string[],
token?: CancellationToken,
): Promise<void> {
const args = [
"-o",
@@ -1495,6 +1684,10 @@ export class CodeQLCliServer implements Disposable {
["pack", "bundle"],
args,
"Bundling pack",
{
runInNewProcess: true,
token,
},
);
}

15
extensions/ql-vscode/src/codeql-cli/query-language.ts
View File

@@ -1,5 +1,5 @@
import type { CodeQLCliServer } from "./cli";
import type { Uri } from "vscode";
import type { CancellationToken, Uri } from "vscode";
import { window } from "vscode";
import {
getLanguageDisplayName,
@@ -50,6 +50,7 @@ export async function findLanguage(
export async function askForLanguage(
cliServer: CodeQLCliServer,
throwOnEmpty = true,
token?: CancellationToken,
): Promise<QueryLanguage | undefined> {
const supportedLanguages = await cliServer.getSupportedLanguages();
@@ -62,10 +63,14 @@ export async function askForLanguage(
}))
.sort((a, b) => a.label.localeCompare(b.label));
const selectedItem = await window.showQuickPick(items, {
placeHolder: "Select target query language",
ignoreFocusOut: true,
});
const selectedItem = await window.showQuickPick(
items,
{
placeHolder: "Select target query language",
ignoreFocusOut: true,
},
token,
);
if (!selectedItem) {
// This only happens if the user cancels the quick pick.
if (throwOnEmpty) {

4
extensions/ql-vscode/src/common/commands.ts
View File

@@ -220,6 +220,7 @@ export type LocalDatabasesCommands = {
// Explorer context menu
"codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
"codeQL.importTestDatabase": (uri: Uri) => Promise<void>;
// Database panel view title commands
"codeQLDatabases.chooseDatabaseFolder": () => Promise<void>;
@@ -272,6 +273,9 @@ export type VariantAnalysisCommands = {
"codeQL.openVariantAnalysisLogs": (
variantAnalysisId: number,
) => Promise<void>;
"codeQLModelAlerts.openVariantAnalysisLogs": (
variantAnalysisId: number,
) => Promise<void>;
"codeQL.openVariantAnalysisView": (
variantAnalysisId: number,
) => Promise<void>;

50
extensions/ql-vscode/src/common/interface-types.ts
View File

@@ -10,10 +10,11 @@ import type {
} from "../variant-analysis/shared/variant-analysis-filter-sort";
import type { ErrorLike } from "../common/errors";
import type { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
import type { Method } from "../model-editor/method";
import type { Method, MethodSignature } from "../model-editor/method";
import type { ModeledMethod } from "../model-editor/modeled-method";
import type {
MethodModelingPanelViewState,
ModelAlertsViewState,
ModelEditorViewState,
} from "../model-editor/shared/view-state";
import type { Mode } from "../model-editor/shared/mode";
@@ -604,6 +605,11 @@ interface OpenModelAlertsViewMessage {
t: "openModelAlertsView";
}
interface RevealInModelAlertsViewMessage {
t: "revealInModelAlertsView";
modeledMethod: ModeledMethod;
}
interface ModelDependencyMessage {
t: "modelDependency";
}
@@ -676,11 +682,12 @@ export type FromModelEditorMessage =
| SetMultipleModeledMethodsMessage
| StartModelEvaluationMessage
| StopModelEvaluationMessage
| OpenModelAlertsViewMessage;
| OpenModelAlertsViewMessage
| RevealInModelAlertsViewMessage;
interface RevealInEditorMessage {
t: "revealInModelEditor";
method: Method;
method: MethodSignature;
}
interface StartModelingMessage {
@@ -726,10 +733,39 @@ export type ToMethodModelingMessage =
| SetInProgressMessage
| SetProcessedByAutoModelMessage;
interface SetModelAlertsMessage {
t: "setModelAlerts";
interface SetModelAlertsViewStateMessage {
t: "setModelAlertsViewState";
viewState: ModelAlertsViewState;
}
export type ToModelAlertsMessage = SetModelAlertsMessage;
interface OpenModelPackMessage {
t: "openModelPack";
path: string;
}
export type FromModelAlertsMessage = CommonFromViewMessages;
interface OpenActionsLogsMessage {
t: "openActionsLogs";
variantAnalysisId: number;
}
interface StopEvaluationRunMessage {
t: "stopEvaluationRun";
}
interface RevealModelMessage {
t: "revealModel";
modeledMethod: ModeledMethod;
}
export type ToModelAlertsMessage =
| SetModelAlertsViewStateMessage
| SetVariantAnalysisMessage
| SetRepoResultsMessage
| RevealModelMessage;
export type FromModelAlertsMessage =
| CommonFromViewMessages
| OpenModelPackMessage
| OpenActionsLogsMessage
| StopEvaluationRunMessage
| RevealInEditorMessage;

4
extensions/ql-vscode/src/common/model-pack-details.ts Normal file
View File

@@ -0,0 +1,4 @@
export interface ModelPackDetails {
name: string;
path: string;
}

2
extensions/ql-vscode/src/common/query-metadata.ts
View File

@@ -1,4 +1,4 @@
const SARIF_RESULTS_QUERY_KINDS = [
export const SARIF_RESULTS_QUERY_KINDS = [
"problem",
"alert",
"path-problem",

9
extensions/ql-vscode/src/common/vscode/abstract-webview.ts
View File

@@ -13,6 +13,8 @@ import { tmpDir } from "../../tmp-dir";
import type { WebviewMessage, WebviewKind } from "./webview-html";
import { getHtmlForWebview } from "./webview-html";
import type { DeepReadonly } from "../readonly";
import { runWithErrorHandling } from "./error-handling";
import { telemetryListener } from "./telemetry";
export type WebviewPanelConfig = {
viewId: string;
@@ -117,7 +119,12 @@ export abstract class AbstractWebview<
);
this.push(
panel.webview.onDidReceiveMessage(
async (e) => this.onMessage(e),
async (e) =>
runWithErrorHandling(
() => this.onMessage(e),
this.app.logger,
telemetryListener,
),
undefined,
),
);

11
extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts
View File

@@ -242,6 +242,17 @@ export class ArchiveFileSystemProvider implements FileSystemProvider {
root = new Directory("");
constructor() {
// When a file system archive is removed from the workspace, we should
// also remove it from our cache.
workspace.onDidChangeWorkspaceFolders((event) => {
for (const removed of event.removed) {
const zipPath = removed.uri.fsPath;
this.archives.delete(zipPath);
}
});
}
// metadata
async stat(uri: Uri): Promise<FileStat> {

57
extensions/ql-vscode/src/common/vscode/commands.ts
View File

@@ -3,18 +3,10 @@ import { commands } from "vscode";
import type { CommandFunction } from "../../packages/commands";
import { CommandManager } from "../../packages/commands";
import type { NotificationLogger } from "../logging";
import {
showAndLogWarningMessage,
showAndLogExceptionWithTelemetry,
} from "../logging";
import { extLogger } from "../logging/vscode";
import { asError, getErrorMessage } from "../../common/helpers-pure";
import { redactableError } from "../../common/errors";
import { UserCancellationException } from "./progress";
import { telemetryListener } from "./telemetry";
import type { AppTelemetry } from "../telemetry";
import { CliError } from "../../codeql-cli/cli-errors";
import { EOL } from "os";
import { runWithErrorHandling } from "./error-handling";
/**
* Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -48,50 +40,9 @@ export function registerCommandWithErrorHandling<
logger: NotificationLogger = extLogger,
telemetry: AppTelemetry | undefined = telemetryListener,
): Disposable {
return commands.registerCommand(commandId, async (...args: Parameters<T>) => {
const startTime = Date.now();
let error: Error | undefined;
try {
return await task(...args);
} catch (e) {
error = asError(e);
const errorMessage = redactableError(error)`${
getErrorMessage(e) || e
} (${commandId})`;
if (e instanceof UserCancellationException) {
// User has cancelled this action manually
if (e.silent) {
void logger.log(errorMessage.fullMessage);
} else {
void showAndLogWarningMessage(logger, errorMessage.fullMessage);
}
} else if (e instanceof CliError) {
const fullMessage = `${e.commandDescription} failed with args:${EOL} ${e.commandArgs.join(" ")}${EOL}${
e.stderr ?? e.cause
}`;
void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
fullMessage,
extraTelemetryProperties: {
command: commandId,
},
});
} else {
// Include the full stack in the error log only.
const fullMessage = errorMessage.fullMessageWithStack;
void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
fullMessage,
extraTelemetryProperties: {
command: commandId,
},
});
}
return undefined;
} finally {
const executionTime = Date.now() - startTime;
telemetryListener?.sendCommandUsage(commandId, executionTime, error);
}
});
return commands.registerCommand(commandId, async (...args: Parameters<T>) =>
runWithErrorHandling(task, logger, telemetry, commandId, ...args),
);
}
/**

81
extensions/ql-vscode/src/common/vscode/error-handling.ts Normal file
View File

@@ -0,0 +1,81 @@
import {
showAndLogWarningMessage,
showAndLogExceptionWithTelemetry,
} from "../logging";
import type { NotificationLogger } from "../logging";
import { extLogger } from "../logging/vscode";
import type { AppTelemetry } from "../telemetry";
import { telemetryListener } from "./telemetry";
import { asError, getErrorMessage } from "../helpers-pure";
import { redactableError } from "../errors";
import { UserCancellationException } from "./progress";
import { CliError } from "../../codeql-cli/cli-errors";
import { EOL } from "os";
/**
* Executes a task with error handling. It provides a uniform way to handle errors.
*
* @template T - A function type that takes an unknown number of arguments and returns a Promise.
* @param {T} task - The task to be executed.
* @param {NotificationLogger} [logger=extLogger] - The logger to use for error reporting.
* @param {AppTelemetry | undefined} [telemetry=telemetryListener] - The telemetry listener to use for error reporting.
* @param {string} [commandId] - The optional command id associated with the task.
* @param {...unknown} args - The arguments to be passed to the task.
* @returns {Promise<unknown>} The result of the task, or undefined if an error occurred.
* @throws {Error} If an error occurs during the execution of the task.
*/
export async function runWithErrorHandling<
T extends (...args: unknown[]) => Promise<unknown>,
>(
task: T,
logger: NotificationLogger = extLogger,
telemetry: AppTelemetry | undefined = telemetryListener,
commandId?: string,
...args: unknown[]
): Promise<unknown> {
const startTime = Date.now();
let error: Error | undefined;
try {
return await task(...args);
} catch (e) {
error = asError(e);
const errorMessage = redactableError(error)`${
getErrorMessage(e) || e
}${commandId ? ` (${commandId})` : ""}`;
const extraTelemetryProperties = commandId
? { command: commandId }
: undefined;
if (e instanceof UserCancellationException) {
// User has cancelled this action manually
if (e.silent) {
void logger.log(errorMessage.fullMessage);
} else {
void showAndLogWarningMessage(logger, errorMessage.fullMessage);
}
} else if (e instanceof CliError) {
const fullMessage = `${e.commandDescription} failed with args:${EOL} ${e.commandArgs.join(" ")}${EOL}${
e.stderr ?? e.cause
}`;
void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
fullMessage,
extraTelemetryProperties,
});
} else {
// Include the full stack in the error log only.
const fullMessage = errorMessage.fullMessageWithStack;
void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
fullMessage,
extraTelemetryProperties,
});
}
return undefined;
} finally {
if (commandId) {
const executionTime = Date.now() - startTime;
telemetryListener?.sendCommandUsage(commandId, executionTime, error);
}
}
}

23
extensions/ql-vscode/src/common/vscode/progress.ts
View File

@@ -85,29 +85,6 @@ export function withProgress<R>(
);
}
export interface ProgressContext {
progress: ProgressCallback;
token: CancellationToken;
}
/**
* Like `withProgress()`, except that the caller is not required to provide a progress context. If
* the caller does provide one, any long-running operations performed by `task` will use the
* supplied progress context. Otherwise, this function wraps `task` in a new progress context with
* the supplied options.
*/
export function withInheritedProgress<R>(
parent: ProgressContext | undefined,
task: ProgressTask<R>,
options: ProgressOptions,
): Thenable<R> {
if (parent !== undefined) {
return task(parent.progress, parent.token);
} else {
return withProgress(task, options);
}
}
/**
* Displays a progress monitor that indicates how much progess has been made
* reading from a stream.

89
extensions/ql-vscode/src/databases/database-fetcher.ts
View File

@@ -1,16 +1,15 @@
import type { Response } from "node-fetch";
import fetch, { AbortError } from "node-fetch";
import { zip } from "zip-a-folder";
import type { InputBoxOptions } from "vscode";
import { Uri, window } from "vscode";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import {
ensureDir,
realpath as fs_realpath,
pathExists,
createWriteStream,
remove,
readdir,
copy,
} from "fs-extra";
import { basename, join } from "path";
import type { Octokit } from "@octokit/rest";
@@ -40,6 +39,7 @@ import type { App } from "../common/app";
import { createFilenameFromString } from "../common/filenames";
import { findDirWithFile } from "../common/files";
import { convertGithubNwoToDatabaseUrl } from "./github-databases/api";
import { ensureZippedSourceLocation } from "./local-databases/database-contents";
// The number of tries to use when generating a unique filename before
// giving up and using a nanoid.
@@ -74,7 +74,7 @@ export class DatabaseFetcher {
this.validateUrl(databaseUrl);
const item = await this.databaseArchiveFetcher(
const item = await this.fetchDatabaseToWorkspaceStorage(
databaseUrl,
{},
undefined,
@@ -247,7 +247,7 @@ export class DatabaseFetcher {
* We only need the actual token string.
*/
const octokitToken = ((await octokit.auth()) as { token: string })?.token;
return await this.databaseArchiveFetcher(
return await this.fetchDatabaseToWorkspaceStorage(
databaseUrl,
{
Accept: "application/zip",
@@ -268,31 +268,35 @@ export class DatabaseFetcher {
}
/**
* Imports a database from a local archive.
* Imports a database from a local archive or a test database that is in a folder
* ending with `.testproj`.
*
* @param databaseUrl the file url of the archive to import
* @param databaseUrl the file url of the archive or directory to import
* @param progress the progress callback
*/
public async importArchiveDatabase(
public async importLocalDatabase(
databaseUrl: string,
progress: ProgressCallback,
): Promise<DatabaseItem | undefined> {
try {
const item = await this.databaseArchiveFetcher(
const origin: DatabaseOrigin = {
type: databaseUrl.endsWith(".testproj") ? "testproj" : "archive",
path: Uri.parse(databaseUrl).fsPath,
};
const item = await this.fetchDatabaseToWorkspaceStorage(
databaseUrl,
{},
undefined,
{
type: "archive",
path: databaseUrl,
},
origin,
progress,
);
if (item) {
await this.app.commands.execute("codeQLDatabases.focus");
void showAndLogInformationMessage(
extLogger,
"Database unzipped and imported successfully.",
origin.type === "testproj"
? "Test database imported successfully."
: "Database unzipped and imported successfully.",
);
}
return item;
@@ -309,10 +313,10 @@ export class DatabaseFetcher {
}
/**
* Fetches an archive database. The database might be on the internet
* Fetches a database into workspace storage. The database might be on the internet
* or in the local filesystem.
*
* @param databaseUrl URL from which to grab the database
* @param databaseUrl URL from which to grab the database. This could be a local archive file, a local directory, or a remote URL.
* @param requestHeaders Headers to send with the request
* @param nameOverride a name for the database that overrides the default
* @param origin the origin of the database
@@ -320,7 +324,7 @@ export class DatabaseFetcher {
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
*/
private async databaseArchiveFetcher(
private async fetchDatabaseToWorkspaceStorage(
databaseUrl: string,
requestHeaders: { [key: string]: string },
nameOverride: string | undefined,
@@ -341,7 +345,11 @@ export class DatabaseFetcher {
const unzipPath = await this.getStorageFolder(databaseUrl, nameOverride);
if (Uri.parse(databaseUrl).scheme === "file") {
await this.readAndUnzip(databaseUrl, unzipPath, progress);
if (origin.type === "testproj") {
await this.copyDatabase(databaseUrl, unzipPath, progress);
} else {
await this.readAndUnzip(databaseUrl, unzipPath, progress);
}
} else {
await this.fetchAndUnzip(
databaseUrl,
@@ -369,7 +377,7 @@ export class DatabaseFetcher {
step: 4,
maxStep: 4,
});
await this.ensureZippedSourceLocation(dbPath);
await ensureZippedSourceLocation(dbPath);
const item = await this.databaseManager.openDatabase(
Uri.file(dbPath),
@@ -402,6 +410,8 @@ export class DatabaseFetcher {
lastName = basename(url.path).substring(0, 250);
if (lastName.endsWith(".zip")) {
lastName = lastName.substring(0, lastName.length - 4);
} else if (lastName.endsWith(".testproj")) {
lastName = lastName.substring(0, lastName.length - 9);
}
}
@@ -448,6 +458,26 @@ export class DatabaseFetcher {
}
}
/**
* Copies a database folder from the file system into the workspace storage.
* @param scrDirURL the original location of the database as a URL string
* @param destDir the location to copy the database to. This should be a folder in the workspace storage.
* @param progress callback to send progress messages to
*/
private async copyDatabase(
srcDirURL: string,
destDir: string,
progress?: ProgressCallback,
) {
progress?.({
maxStep: 10,
step: 9,
message: `Copying database ${basename(destDir)} into the workspace`,
});
await ensureDir(destDir);
await copy(Uri.parse(srcDirURL).fsPath, destDir);
}
private async readAndUnzip(
zipUrl: string,
unzipPath: string,
@@ -582,27 +612,4 @@ export class DatabaseFetcher {
}
throw new Error(`${errorMessage}.\n\nReason: ${msg}`);
}
/**
* Databases created by the old odasa tool will not have a zipped
* source location. However, this extension works better if sources
* are zipped.
*
* This function ensures that the source location is zipped. If the
* `src` folder exists and the `src.zip` file does not, the `src`
* folder will be zipped and then deleted.
*
* @param databasePath The full path to the unzipped database
*/
private async ensureZippedSourceLocation(
databasePath: string,
): Promise<void> {
const srcFolderPath = join(databasePath, "src");
const srcZipPath = `${srcFolderPath}.zip`;
if ((await pathExists(srcFolderPath)) && !(await pathExists(srcZipPath))) {
await zip(srcFolderPath, srcZipPath);
await remove(srcFolderPath);
}
}
}

133
extensions/ql-vscode/src/databases/local-databases-ui.ts
View File

@@ -16,7 +16,6 @@ import {
ThemeIcon,
ThemeColor,
workspace,
ProgressLocation,
} from "vscode";
import { pathExists, stat, readdir, remove } from "fs-extra";
@@ -25,13 +24,9 @@ import type {
DatabaseItem,
DatabaseManager,
} from "./local-databases";
import type {
ProgressCallback,
ProgressContext,
} from "../common/vscode/progress";
import type { ProgressCallback } from "../common/vscode/progress";
import {
UserCancellationException,
withInheritedProgress,
withProgress,
} from "../common/vscode/progress";
import {
@@ -141,7 +136,8 @@ class DatabaseTreeDataProvider
item.iconPath = new ThemeIcon("error", new ThemeColor("errorForeground"));
}
item.tooltip = element.databaseUri.fsPath;
item.description = element.language;
item.description =
element.language + (element.origin?.type === "testproj" ? " (test)" : "");
return item;
}
@@ -278,6 +274,7 @@ export class DatabaseUI extends DisposableObject {
this.handleChooseDatabaseInternet.bind(this),
"codeQL.chooseDatabaseGithub": this.handleChooseDatabaseGithub.bind(this),
"codeQL.setCurrentDatabase": this.handleSetCurrentDatabase.bind(this),
"codeQL.importTestDatabase": this.handleImportTestDatabase.bind(this),
"codeQL.setDefaultTourDatabase":
this.handleSetDefaultTourDatabase.bind(this),
"codeQL.upgradeCurrentDatabase":
@@ -327,10 +324,9 @@ export class DatabaseUI extends DisposableObject {
private async chooseDatabaseFolder(
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
try {
await this.chooseAndSetDatabase(true, { progress, token });
await this.chooseAndSetDatabase(true, progress);
} catch (e) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
@@ -344,8 +340,8 @@ export class DatabaseUI extends DisposableObject {
private async handleChooseDatabaseFolder(): Promise<void> {
return withProgress(
async (progress, token) => {
await this.chooseDatabaseFolder(progress, token);
async (progress) => {
await this.chooseDatabaseFolder(progress);
},
{
title: "Adding database from folder",
@@ -355,8 +351,8 @@ export class DatabaseUI extends DisposableObject {
private async handleChooseDatabaseFolderFromPalette(): Promise<void> {
return withProgress(
async (progress, token) => {
await this.chooseDatabaseFolder(progress, token);
async (progress) => {
await this.chooseDatabaseFolder(progress);
},
{
title: "Choose a Database from a Folder",
@@ -497,10 +493,9 @@ export class DatabaseUI extends DisposableObject {
private async chooseDatabaseArchive(
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
try {
await this.chooseAndSetDatabase(false, { progress, token });
await this.chooseAndSetDatabase(false, progress);
} catch (e: unknown) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
@@ -514,8 +509,8 @@ export class DatabaseUI extends DisposableObject {
private async handleChooseDatabaseArchive(): Promise<void> {
return withProgress(
async (progress, token) => {
await this.chooseDatabaseArchive(progress, token);
async (progress) => {
await this.chooseDatabaseArchive(progress);
},
{
title: "Adding database from archive",
@@ -525,8 +520,8 @@ export class DatabaseUI extends DisposableObject {
private async handleChooseDatabaseArchiveFromPalette(): Promise<void> {
return withProgress(
async (progress, token) => {
await this.chooseDatabaseArchive(progress, token);
async (progress) => {
await this.chooseDatabaseArchive(progress);
},
{
title: "Choose a Database from an Archive",
@@ -697,7 +692,7 @@ export class DatabaseUI extends DisposableObject {
try {
// Assume user has selected an archive if the file has a .zip extension
if (uri.path.endsWith(".zip")) {
await this.databaseFetcher.importArchiveDatabase(
await this.databaseFetcher.importLocalDatabase(
uri.toString(true),
progress,
);
@@ -721,6 +716,56 @@ export class DatabaseUI extends DisposableObject {
);
}
private async handleImportTestDatabase(uri: Uri): Promise<void> {
return withProgress(
async (progress) => {
try {
if (!uri.path.endsWith(".testproj")) {
throw new Error(
"Please select a valid test database to import. Test databases end with `.testproj`.",
);
}
// Check if the database is already in the workspace. If
// so, delete it first before importing the new one.
const existingItem = this.databaseManager.findTestDatabase(uri);
const baseName = basename(uri.fsPath);
if (existingItem !== undefined) {
progress({
maxStep: 9,
step: 1,
message: `Removing existing test database ${baseName}`,
});
await this.databaseManager.removeDatabaseItem(existingItem);
}
await this.databaseFetcher.importLocalDatabase(
uri.toString(true),
progress,
);
if (existingItem !== undefined) {
progress({
maxStep: 9,
step: 9,
message: `Successfully re-imported ${baseName}`,
});
}
} catch (e) {
// rethrow and let this be handled by default error handling.
throw new Error(
`Could not set database to ${basename(
uri.fsPath,
)}. Reason: ${getErrorMessage(e)}`,
);
}
},
{
title: "(Re-)importing test database from directory",
},
);
}
private async handleRemoveDatabase(
databaseItems: DatabaseItem[],
): Promise<void> {
@@ -776,9 +821,8 @@ export class DatabaseUI extends DisposableObject {
*/
public async getDatabaseItem(
progress: ProgressCallback,
token: CancellationToken,
): Promise<DatabaseItem | undefined> {
return await this.getDatabaseItemInternal({ progress, token });
return await this.getDatabaseItemInternal(progress);
}
/**
@@ -791,10 +835,10 @@ export class DatabaseUI extends DisposableObject {
* notification if it tries to perform any long-running operations.
*/
private async getDatabaseItemInternal(
progressContext: ProgressContext | undefined,
progress: ProgressCallback | undefined,
): Promise<DatabaseItem | undefined> {
if (this.databaseManager.currentDatabaseItem === undefined) {
progressContext?.progress({
progress?.({
maxStep: 2,
step: 1,
message: "Choosing database",
@@ -921,37 +965,28 @@ export class DatabaseUI extends DisposableObject {
*/
private async chooseAndSetDatabase(
byFolder: boolean,
progress: ProgressContext | undefined,
progress: ProgressCallback,
): Promise<DatabaseItem | undefined> {
const uri = await chooseDatabaseDir(byFolder);
if (!uri) {
return undefined;
}
return await withInheritedProgress(
progress,
async (progress) => {
if (byFolder) {
const fixedUri = await this.fixDbUri(uri);
// we are selecting a database folder
return await this.databaseManager.openDatabase(fixedUri, {
type: "folder",
});
} else {
// we are selecting a database archive. Must unzip into a workspace-controlled area
// before importing.
return await this.databaseFetcher.importArchiveDatabase(
uri.toString(true),
progress,
);
}
},
{
location: ProgressLocation.Notification,
cancellable: true,
title: "Opening database",
},
);
if (byFolder && !uri.fsPath.endsWith("testproj")) {
const fixedUri = await this.fixDbUri(uri);
// we are selecting a database folder
return await this.databaseManager.openDatabase(fixedUri, {
type: "folder",
});
} else {
// we are selecting a database archive or a testproj.
// Unzip archives (if an archive) and copy into a workspace-controlled area
// before importing.
return await this.databaseFetcher.importLocalDatabase(
uri.toString(true),
progress,
);
}
}
/**

26
extensions/ql-vscode/src/databases/local-databases/database-contents.ts
View File

@@ -1,4 +1,7 @@
import { pathExists, remove } from "fs-extra";
import { join } from "path/posix";
import type { Uri } from "vscode";
import { zip } from "zip-a-folder";
/**
* The layout of the database.
@@ -28,3 +31,26 @@ export interface DatabaseContents {
export interface DatabaseContentsWithDbScheme extends DatabaseContents {
dbSchemeUri: Uri; // Always present
}
/**
* Databases created by the old odasa tool will not have a zipped
* source location. However, this extension works better if sources
* are zipped.
*
* This function ensures that the source location is zipped. If the
* `src` folder exists and the `src.zip` file does not, the `src`
* folder will be zipped and then deleted.
*
* @param databasePath The full path to the unzipped database
*/
export async function ensureZippedSourceLocation(
databasePath: string,
): Promise<void> {
const srcFolderPath = join(databasePath, "src");
const srcZipPath = `${srcFolderPath}.zip`;
if ((await pathExists(srcFolderPath)) && !(await pathExists(srcZipPath))) {
await zip(srcFolderPath, srcZipPath);
await remove(srcFolderPath);
}
}

112
extensions/ql-vscode/src/databases/local-databases/database-manager.ts
View File

@@ -18,7 +18,10 @@ import {
import { join } from "path";
import type { FullDatabaseOptions } from "./database-options";
import { DatabaseItemImpl } from "./database-item-impl";
import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
import {
showBinaryChoiceDialog,
showNeverAskAgainDialog,
} from "../../common/vscode/dialog";
import {
getFirstWorkspaceFolder,
isFolderAlreadyInWorkspace,
@@ -32,7 +35,7 @@ import { QlPackGenerator } from "../../local-queries/qlpack-generator";
import { asError, getErrorMessage } from "../../common/helpers-pure";
import type { DatabaseItem, PersistedDatabaseItem } from "./database-item";
import { redactableError } from "../../common/errors";
import { remove } from "fs-extra";
import { copy, remove, stat } from "fs-extra";
import { containsPath } from "../../common/files";
import type { DatabaseChangedEvent } from "./database-events";
import { DatabaseEventKind } from "./database-events";
@@ -40,6 +43,8 @@ import { DatabaseResolver } from "./database-resolver";
import { telemetryListener } from "../../common/vscode/telemetry";
import type { LanguageContextStore } from "../../language-context-store";
import type { DatabaseOrigin } from "./database-origin";
import {} from "../database-fetcher";
import { ensureZippedSourceLocation } from "./database-contents";
/**
* The name of the key in the workspaceState dictionary in which we
@@ -120,6 +125,7 @@ export class DatabaseManager extends DisposableObject {
super();
qs.onStart(this.reregisterDatabases.bind(this));
qs.onQueryRunStarting(this.maybeReimportTestDatabase.bind(this));
this.push(
this.languageContext.onLanguageContextChanged(async () => {
@@ -165,6 +171,108 @@ export class DatabaseManager extends DisposableObject {
);
}
/**
* Finds a test database that was originally imported from `uri`.
* A test database is creeated by the `codeql test run` command
* and ends with `.testproj`.
* @param uri The original location of the database
* @returns The first database item found that matches the uri
*/
public findTestDatabase(uri: vscode.Uri): DatabaseItem | undefined {
const originPath = uri.fsPath;
for (const item of this._databaseItems) {
if (item.origin?.type === "testproj" && item.origin.path === originPath) {
return item;
}
}
return undefined;
}
public async maybeReimportTestDatabase(
databaseUri: vscode.Uri,
forceImport = false,
): Promise<void> {
const res = await this.isTestDatabaseOutdated(databaseUri);
if (!res) {
return;
}
const doit =
forceImport ||
(await showBinaryChoiceDialog(
"This test database is outdated. Do you want to reimport it?",
));
if (doit) {
await this.reimportTestDatabase(databaseUri);
}
}
/**
* Checks if the origin of the imported database is newer.
* The imported database must be a test database.
* @param databaseUri the URI of the imported database to check
* @returns true if both databases exist and the origin database is newer.
*/
private async isTestDatabaseOutdated(
databaseUri: vscode.Uri,
): Promise<boolean> {
const dbItem = this.findDatabaseItem(databaseUri);
if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
return false;
}
// Compare timestmps of the codeql-database.yml files of the original and the
// imported databases.
const originDbYml = join(dbItem.origin.path, "codeql-database.yml");
const importedDbYml = join(
dbItem.databaseUri.fsPath,
"codeql-database.yml",
);
let originStat;
try {
originStat = await stat(originDbYml);
} catch (e) {
// if there is an error here, assume that the origin database
// is no longer available. Safely ignore and do not try to re-import.
return false;
}
try {
const importedStat = await stat(importedDbYml);
return originStat.mtimeMs > importedStat.mtimeMs;
} catch (e) {
// If either of the files does not exist, we assume the origin is newer.
// This shouldn't happen unless the user manually deleted one of the files.
return true;
}
}
/**
* Reimport the specified imported database from its origin.
* The imported databsae must be a testproj database.
*
* @param databaseUri the URI of the imported database to reimport
*/
private async reimportTestDatabase(databaseUri: vscode.Uri): Promise<void> {
const dbItem = this.findDatabaseItem(databaseUri);
if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
throw new Error(`Database ${databaseUri} is not a testproj.`);
}
await this.removeDatabaseItem(dbItem);
await copy(dbItem.origin.path, databaseUri.fsPath);
await ensureZippedSourceLocation(databaseUri.fsPath);
const newDbItem = new DatabaseItemImpl(databaseUri, dbItem.contents, {
dateAdded: Date.now(),
language: dbItem.language,
origin: dbItem.origin,
extensionManagedLocation: dbItem.extensionManagedLocation,
});
await this.addDatabaseItem(newDbItem);
await this.setCurrentDatabaseItem(newDbItem);
}
/**
* Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
* the list.

8
extensions/ql-vscode/src/databases/local-databases/database-origin.ts
View File

@@ -24,9 +24,15 @@ interface DatabaseOriginDebugger {
type: "debugger";
}
interface DatabaseOriginTestProj {
type: "testproj";
path: string;
}
export type DatabaseOrigin =
| DatabaseOriginFolder
| DatabaseOriginArchive
| DatabaseOriginGitHub
| DatabaseOriginInternet
| DatabaseOriginDebugger;
| DatabaseOriginDebugger
| DatabaseOriginTestProj;

24
extensions/ql-vscode/src/local-queries/local-queries.ts
View File

@@ -2,7 +2,10 @@ import type {
ProgressCallback,
ProgressUpdate,
} from "../common/vscode/progress";
import { withProgress } from "../common/vscode/progress";
import {
UserCancellationException,
withProgress,
} from "../common/vscode/progress";
import type { CancellationToken, Range, TabInputText } from "vscode";
import { CancellationTokenSource, Uri, window } from "vscode";
import {
@@ -292,14 +295,8 @@ export class LocalQueries extends DisposableObject {
private async quickQuery(): Promise<void> {
await withProgress(
async (progress, token) =>
displayQuickQuery(
this.app,
this.cliServer,
this.databaseUI,
progress,
token,
),
async (progress) =>
displayQuickQuery(this.app, this.cliServer, this.databaseUI, progress),
{
title: "Run Quick Query",
},
@@ -445,7 +442,7 @@ export class LocalQueries extends DisposableObject {
// If no databaseItem is specified, use the database currently selected in the Databases UI
databaseItem =
databaseItem ?? (await this.databaseUI.getDatabaseItem(progress, token));
databaseItem ?? (await this.databaseUI.getDatabaseItem(progress));
if (databaseItem === undefined) {
throw new Error("Can't run query without a selected database");
}
@@ -498,7 +495,12 @@ export class LocalQueries extends DisposableObject {
// to unify both error handling paths.
const err = asError(e);
await localQueryRun.fail(err);
throw e;
if (token.isCancellationRequested) {
throw new UserCancellationException(err.message, true);
} else {
throw e;
}
}
} finally {
source.dispose();

4
extensions/ql-vscode/src/local-queries/quick-query.ts
View File

@@ -1,7 +1,6 @@
import { ensureDir, writeFile, pathExists, readFile } from "fs-extra";
import { dump, load } from "js-yaml";
import { basename, join } from "path";
import type { CancellationToken } from "vscode";
import { window as Window, workspace, Uri } from "vscode";
import { LSPErrorCodes, ResponseError } from "vscode-languageclient";
import type { CodeQLCliServer } from "../codeql-cli/cli";
@@ -56,7 +55,6 @@ export async function displayQuickQuery(
cliServer: CodeQLCliServer,
databaseUI: DatabaseUI,
progress: ProgressCallback,
token: CancellationToken,
) {
try {
// If there is already a quick query open, don't clobber it, just
@@ -111,7 +109,7 @@ export async function displayQuickQuery(
}
// We're going to infer which qlpack to use from the current database
const dbItem = await databaseUI.getDatabaseItem(progress, token);
const dbItem = await databaseUI.getDatabaseItem(progress);
if (dbItem === undefined) {
throw new Error("Can't start quick query without a selected database");
}

3
extensions/ql-vscode/src/model-editor/extension-pack-metadata.schema.json
View File

@@ -111,6 +111,9 @@
"description": {
"type": "string"
},
"import": {
"type": "string"
},
"from": {
"type": "string"
}

21
extensions/ql-vscode/src/model-editor/languages/csharp/index.ts Normal file
View File

@@ -0,0 +1,21 @@
import type { ModelsAsDataLanguage } from "../models-as-data";
import { staticLanguage } from "../static";
export const csharp: ModelsAsDataLanguage = {
...staticLanguage,
predicates: {
...staticLanguage.predicates,
sink: {
...staticLanguage.predicates.sink,
},
source: {
...staticLanguage.predicates.source,
supportedKinds: [
...staticLanguage.predicates.source.supportedKinds,
// https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L122-L123
"file-write",
"windows-registry",
],
},
},
};

42
extensions/ql-vscode/src/model-editor/languages/java/index.ts Normal file
View File

@@ -0,0 +1,42 @@
import type { ModelsAsDataLanguage } from "../models-as-data";
import { staticLanguage } from "../static";
export const java: ModelsAsDataLanguage = {
...staticLanguage,
predicates: {
...staticLanguage.predicates,
sink: {
...staticLanguage.predicates.sink,
supportedKinds: [
...staticLanguage.predicates.sink.supportedKinds,
// https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L32-L37
"bean-validation",
"fragment-injection",
"groovy-injection",
"hostname-verification",
"information-leak",
"intent-redirection",
"jexl-injection",
"jndi-injection",
"mvel-injection",
"notification",
"ognl-injection",
"pending-intents",
"response-splitting",
"trust-boundary-violation",
"template-injection",
"xpath-injection",
"xslt-injection",
],
},
source: {
...staticLanguage.predicates.source,
supportedKinds: [
...staticLanguage.predicates.source.supportedKinds,
// https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L120-L121
"android-external-storage-dir",
"contentprovider",
],
},
},
};

7
extensions/ql-vscode/src/model-editor/languages/languages.ts
View File

@@ -3,13 +3,14 @@ import type {
ModelsAsDataLanguage,
ModelsAsDataLanguagePredicates,
} from "./models-as-data";
import { csharp } from "./csharp";
import { java } from "./java";
import { python } from "./python";
import { ruby } from "./ruby";
import { staticLanguage } from "./static";
const languages: Partial<Record<QueryLanguage, ModelsAsDataLanguage>> = {
[QueryLanguage.CSharp]: staticLanguage,
[QueryLanguage.Java]: staticLanguage,
[QueryLanguage.CSharp]: csharp,
[QueryLanguage.Java]: java,
[QueryLanguage.Python]: python,
[QueryLanguage.Ruby]: ruby,
};

5
extensions/ql-vscode/src/model-editor/languages/python/access-paths.ts
View File

@@ -112,7 +112,10 @@ export function pythonPath(
export function pythonEndpointType(
method: Omit<MethodDefinition, "endpointType">,
): EndpointType {
if (method.methodParameters.startsWith("(self,")) {
if (
method.methodParameters.startsWith("(self,") ||
method.methodParameters === "(self)"
) {
return EndpointType.Method;
}
return EndpointType.Function;

7
extensions/ql-vscode/src/model-editor/languages/shared.ts
View File

@@ -6,10 +6,13 @@ export const sharedExtensiblePredicates = {
};
export const sharedKinds = {
source: ["local", "remote"],
// https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L118-L119
source: ["local", "remote", "file", "commandargs", "database", "environment"],
// Bhttps://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L28-L31
sink: [
"code-injection",
"command-injection",
"environment-injection",
"file-content-store",
"html-injection",
"js-injection",
@@ -20,6 +23,8 @@ export const sharedKinds = {
"sql-injection",
"url-redirection",
],
// https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L142-L143
summary: ["taint", "value"],
// https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L155-L156
neutral: ["summary", "source", "sink"],
};

4
extensions/ql-vscode/src/model-editor/languages/static/index.ts
View File

@@ -10,7 +10,7 @@ function readRowToMethod(row: DataTuple[]): string {
return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
}
export const staticLanguage: ModelsAsDataLanguage = {
export const staticLanguage = {
createMethodSignature: ({
packageName,
typeName,
@@ -168,4 +168,4 @@ export const staticLanguage: ModelsAsDataLanguage = {
argumentsList.length > 0 ? argumentsList[0].path : "Argument[this]",
};
},
};
} satisfies ModelsAsDataLanguage;

4
extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts
View File

@@ -6,7 +6,7 @@ import { telemetryListener } from "../../common/vscode/telemetry";
import { showAndLogExceptionWithTelemetry } from "../../common/logging/notifications";
import type { App } from "../../common/app";
import { redactableError } from "../../common/errors";
import type { Method } from "../method";
import type { Method, MethodSignature } from "../method";
import type { ModelingStore } from "../modeling-store";
import { AbstractWebviewViewProvider } from "../../common/vscode/abstract-webview-view-provider";
import { assertNever } from "../../common/helpers-pure";
@@ -163,7 +163,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
}
}
private async revealInModelEditor(method: Method): Promise<void> {
private async revealInModelEditor(method: MethodSignature): Promise<void> {
if (!this.databaseItem) {
return;
}

100
extensions/ql-vscode/src/model-editor/model-alerts/alert-processor.ts Normal file
View File

@@ -0,0 +1,100 @@
import type { AnalysisAlert } from "../../variant-analysis/shared/analysis-result";
import type { ModeledMethod } from "../modeled-method";
import { EndpointType } from "../method";
import type { ModelAlerts } from "./model-alerts";
import type {
VariantAnalysis,
VariantAnalysisScannedRepositoryResult,
} from "../../variant-analysis/shared/variant-analysis";
/**
* Calculate which model has contributed to each alert.
* @param alerts The alerts to process.
* @param repoResults The analysis results for each repo.
* @returns The alerts grouped by modeled method.
*/
export function calculateModelAlerts(
variantAnalysis: VariantAnalysis,
repoResults: VariantAnalysisScannedRepositoryResult[],
): ModelAlerts[] {
// For now we just return some mock data, but once we have provenance information
// we'll be able to calculate this properly based on the alerts that are passed in
// and potentially some other information.
const modelAlerts: ModelAlerts[] = [];
const repoMap = new Map<number, string>();
for (const scannedRepo of variantAnalysis.scannedRepos || []) {
repoMap.set(scannedRepo.repository.id, scannedRepo.repository.fullName);
}
for (const [i, repoResult] of repoResults.entries()) {
const results = repoResult.interpretedResults || [];
const repository = {
id: repoResult.repositoryId,
fullName: repoMap.get(repoResult.repositoryId) || "",
};
const alerts = results.map(() => {
return {
alert: createMockAlert(),
repository,
};
});
modelAlerts.push({
model: createModeledMethod(i.toString()),
alerts,
});
}
return modelAlerts;
}
function createModeledMethod(suffix: string): ModeledMethod {
return {
libraryVersion: "1.6.0",
signature: `org.sql2o.Connection#createQuery${suffix}(String)`,
endpointType: EndpointType.Method,
packageName: "org.sql2o",
typeName: "Connection",
methodName: `createQuery${suffix}`,
methodParameters: "(String)",
type: "sink",
input: "Argument[0]",
kind: "path-injection",
provenance: "manual",
};
}
function createMockAlert(): AnalysisAlert {
return {
message: {
tokens: [
{
t: "text",
text: "This is an empty block.",
},
],
},
shortDescription: "This is an empty block.",
fileLink: {
fileLinkPrefix:
"https://github.com/expressjs/express/blob/33e8dc303af9277f8a7e4f46abfdcb5e72f6797b",
filePath: "test/app.options.js",
},
severity: "Warning",
codeSnippet: {
startLine: 10,
endLine: 14,
text: " app.del('/', function(){});\n app.get('/users', function(req, res){});\n app.put('/users', function(req, res){});\n\n request(app)\n",
},
highlightedRegion: {
startLine: 12,
startColumn: 41,
endLine: 12,
endColumn: 43,
},
codeFlows: [],
};
}

159
extensions/ql-vscode/src/model-editor/model-alerts/model-alerts-view.ts
View File

@@ -1,4 +1,4 @@
import { ViewColumn } from "vscode";
import { Uri, ViewColumn } from "vscode";
import type { WebviewPanelConfig } from "../../common/vscode/abstract-webview";
import { AbstractWebview } from "../../common/vscode/abstract-webview";
import { assertNever } from "../../common/helpers-pure";
@@ -11,6 +11,17 @@ import type { App } from "../../common/app";
import { redactableError } from "../../common/errors";
import { extLogger } from "../../common/logging/vscode";
import { showAndLogExceptionWithTelemetry } from "../../common/logging";
import type { ModelingEvents } from "../modeling-events";
import type { ModelingStore } from "../modeling-store";
import type { DatabaseItem } from "../../databases/local-databases";
import type { ExtensionPack } from "../shared/extension-pack";
import type {
VariantAnalysis,
VariantAnalysisScannedRepositoryResult,
} from "../../variant-analysis/shared/variant-analysis";
import type { AppEvent, AppEventEmitter } from "../../common/events";
import type { ModeledMethod } from "../modeled-method";
import type { MethodSignature } from "../method";
export class ModelAlertsView extends AbstractWebview<
ToModelAlertsMessage,
@@ -18,15 +29,36 @@ export class ModelAlertsView extends AbstractWebview<
> {
public static readonly viewType = "codeQL.modelAlerts";
public constructor(app: App) {
public readonly onEvaluationRunStopClicked: AppEvent<void>;
private readonly onEvaluationRunStopClickedEventEmitter: AppEventEmitter<void>;
public constructor(
app: App,
private readonly modelingEvents: ModelingEvents,
private readonly modelingStore: ModelingStore,
private readonly dbItem: DatabaseItem,
private readonly extensionPack: ExtensionPack,
) {
super(app);
this.registerToModelingEvents();
this.onEvaluationRunStopClickedEventEmitter = this.push(
app.createEventEmitter<void>(),
);
this.onEvaluationRunStopClicked =
this.onEvaluationRunStopClickedEventEmitter.event;
}
public async showView() {
public async showView(
reposResults: VariantAnalysisScannedRepositoryResult[],
) {
const panel = await this.getPanel();
panel.reveal(undefined, true);
await this.waitForPanelLoaded();
await this.setViewState();
await this.updateReposResults(reposResults);
}
protected async getPanelConfig(): Promise<WebviewPanelConfig> {
@@ -40,7 +72,7 @@ export class ModelAlertsView extends AbstractWebview<
}
protected onPanelDispose(): void {
// Nothing to dispose
this.modelingStore.updateIsModelAlertsViewOpen(this.dbItem, false);
}
protected async onMessage(msg: FromModelAlertsMessage): Promise<void> {
@@ -60,8 +92,127 @@ export class ModelAlertsView extends AbstractWebview<
)`Unhandled error in model alerts view: ${msg.error.message}`,
);
break;
case "openModelPack":
await this.app.commands.execute("revealInExplorer", Uri.file(msg.path));
break;
case "openActionsLogs":
await this.app.commands.execute(
"codeQLModelAlerts.openVariantAnalysisLogs",
msg.variantAnalysisId,
);
break;
case "stopEvaluationRun":
await this.stopEvaluationRun();
break;
case "revealInModelEditor":
await this.revealInModelEditor(msg.method);
break;
default:
assertNever(msg);
}
}
private async setViewState(): Promise<void> {
await this.postMessage({
t: "setModelAlertsViewState",
viewState: {
title: this.extensionPack.name,
},
});
}
public async updateVariantAnalysis(
variantAnalysis: VariantAnalysis,
): Promise<void> {
if (!this.isShowingPanel) {
return;
}
await this.postMessage({
t: "setVariantAnalysis",
variantAnalysis,
});
}
public async updateRepoResults(
repositoryResult: VariantAnalysisScannedRepositoryResult,
): Promise<void> {
if (!this.isShowingPanel) {
return;
}
await this.postMessage({
t: "setRepoResults",
repoResults: [repositoryResult],
});
}
public async updateReposResults(
repoResults: VariantAnalysisScannedRepositoryResult[],
): Promise<void> {
if (!this.isShowingPanel) {
return;
}
await this.postMessage({
t: "setRepoResults",
repoResults,
});
}
public async focusView(): Promise<void> {
this.panel?.reveal();
}
private registerToModelingEvents() {
this.push(
this.modelingEvents.onFocusModelAlertsView(async (event) => {
if (event.dbUri === this.dbItem.databaseUri.toString()) {
await this.focusView();
}
}),
);
this.push(
this.modelingEvents.onDbClosed(async (event) => {
if (event === this.dbItem.databaseUri.toString()) {
this.dispose();
}
}),
);
this.push(
this.modelingEvents.onRevealInModelAlertsView(async (event) => {
if (event.dbUri === this.dbItem.databaseUri.toString()) {
await this.revealMethod(event.modeledMethod);
}
}),
);
}
private async stopEvaluationRun() {
this.onEvaluationRunStopClickedEventEmitter.fire();
}
private async revealInModelEditor(method: MethodSignature): Promise<void> {
if (!this.dbItem) {
return;
}
this.modelingEvents.fireRevealInModelEditorEvent(
this.dbItem.databaseUri.toString(),
method,
);
}
private async revealMethod(method: ModeledMethod): Promise<void> {
const panel = await this.getPanel();
panel?.reveal();
await this.postMessage({
t: "revealModel",
modeledMethod: method,
});
}
}

13
extensions/ql-vscode/src/model-editor/model-alerts/model-alerts.ts Normal file
View File

@@ -0,0 +1,13 @@
import type { AnalysisAlert } from "../../variant-analysis/shared/analysis-result";
import type { ModeledMethod } from "../modeled-method";
export interface ModelAlerts {
model: ModeledMethod;
alerts: Array<{
alert: AnalysisAlert;
repository: {
id: number;
fullName: string;
};
}>;
}

8
extensions/ql-vscode/src/model-editor/model-editor-view.ts
View File

@@ -36,7 +36,7 @@ import {
externalApiQueriesProgressMaxStep,
runModelEditorQueries,
} from "./model-editor-queries";
import type { Method } from "./method";
import type { MethodSignature } from "./method";
import type { ModeledMethod } from "./modeled-method";
import type { ExtensionPack } from "./shared/extension-pack";
import type { ModelConfigListener } from "../config";
@@ -133,6 +133,7 @@ export class ModelEditorView extends AbstractWebview<
this.variantAnalysisManager,
databaseItem,
language,
this.extensionPack,
this.updateModelEvaluationRun.bind(this),
);
this.push(this.modelEvaluator);
@@ -384,6 +385,9 @@ export class ModelEditorView extends AbstractWebview<
case "openModelAlertsView":
await this.modelEvaluator.openModelAlertsView();
break;
case "revealInModelAlertsView":
await this.modelEvaluator.revealInModelAlertsView(msg.modeledMethod);
break;
case "telemetry":
telemetryListener?.sendUIInteraction(msg.action);
break;
@@ -431,7 +435,7 @@ export class ModelEditorView extends AbstractWebview<
this.panel?.reveal();
}
public async revealMethod(method: Method): Promise<void> {
public async revealMethod(method: MethodSignature): Promise<void> {
this.panel?.reveal();
await this.postMessage({

134
extensions/ql-vscode/src/model-editor/model-evaluator.ts
View File

@@ -13,12 +13,15 @@ import {
UserCancellationException,
withProgress,
} from "../common/vscode/progress";
import { VariantAnalysisScannedRepositoryDownloadStatus } from "../variant-analysis/shared/variant-analysis";
import type { VariantAnalysis } from "../variant-analysis/shared/variant-analysis";
import type { CancellationToken } from "vscode";
import { CancellationTokenSource } from "vscode";
import type { QlPackDetails } from "../variant-analysis/ql-pack-details";
import type { App } from "../common/app";
import { ModelAlertsView } from "./model-alerts/model-alerts-view";
import type { ExtensionPack } from "./shared/extension-pack";
import type { ModeledMethod } from "./modeled-method";
export class ModelEvaluator extends DisposableObject {
// Cancellation token source to allow cancelling of the current run
@@ -26,6 +29,8 @@ export class ModelEvaluator extends DisposableObject {
// submitted, we use the variant analysis manager's cancellation support.
private cancellationSource: CancellationTokenSource;
private modelAlertsView: ModelAlertsView | undefined;
public constructor(
private readonly app: App,
private readonly cliServer: CodeQLCliServer,
@@ -34,7 +39,8 @@ export class ModelEvaluator extends DisposableObject {
private readonly variantAnalysisManager: VariantAnalysisManager,
private readonly dbItem: DatabaseItem,
private readonly language: QueryLanguage,
private readonly updateView: (
private readonly extensionPack: ExtensionPack,
private readonly updateModelEditorView: (
run: ModelEvaluationRunState,
) => Promise<void>,
) {
@@ -58,6 +64,7 @@ export class ModelEvaluator extends DisposableObject {
this.app.logger,
this.cliServer,
this.language,
this.cancellationSource.token,
);
if (!qlPack) {
@@ -107,8 +114,59 @@ export class ModelEvaluator extends DisposableObject {
}
public async openModelAlertsView() {
const view = new ModelAlertsView(this.app);
await view.showView();
if (this.modelingStore.isModelAlertsViewOpen(this.dbItem)) {
this.modelingEvents.fireFocusModelAlertsViewEvent(
this.dbItem.databaseUri.toString(),
);
return;
} else {
this.modelingStore.updateIsModelAlertsViewOpen(this.dbItem, true);
this.modelAlertsView = new ModelAlertsView(
this.app,
this.modelingEvents,
this.modelingStore,
this.dbItem,
this.extensionPack,
);
this.modelAlertsView.onEvaluationRunStopClicked(async () => {
await this.stopEvaluation();
});
// There should be a variant analysis available at this point, as the
// view can only opened when the variant analysis is submitted.
const evaluationRun = this.modelingStore.getModelEvaluationRun(
this.dbItem,
);
if (!evaluationRun) {
throw new Error("No evaluation run available");
}
const variantAnalysis =
await this.getVariantAnalysisForRun(evaluationRun);
if (!variantAnalysis) {
throw new Error("No variant analysis available");
}
const reposResults =
this.variantAnalysisManager.getLoadedResultsForVariantAnalysis(
variantAnalysis.id,
);
await this.modelAlertsView.showView(reposResults);
await this.modelAlertsView.updateVariantAnalysis(variantAnalysis);
}
}
public async revealInModelAlertsView(modeledMethod: ModeledMethod) {
if (!this.modelingStore.isModelAlertsViewOpen(this.dbItem)) {
await this.openModelAlertsView();
}
this.modelingEvents.fireRevealInModelAlertsViewEvent(
this.dbItem.databaseUri.toString(),
modeledMethod,
);
}
private registerToModelingEvents() {
@@ -116,7 +174,7 @@ export class ModelEvaluator extends DisposableObject {
this.modelingEvents.onModelEvaluationRunChanged(async (event) => {
if (event.dbUri === this.dbItem.databaseUri.toString()) {
if (!event.evaluationRun) {
await this.updateView({
await this.updateModelEditorView({
isPreparing: false,
variantAnalysis: undefined,
});
@@ -128,7 +186,7 @@ export class ModelEvaluator extends DisposableObject {
isPreparing: event.evaluationRun.isPreparing,
variantAnalysis,
};
await this.updateView(run);
await this.updateModelEditorView(run);
}
}
}),
@@ -205,14 +263,76 @@ export class ModelEvaluator extends DisposableObject {
this.variantAnalysisManager.onVariantAnalysisStatusUpdated(
async (variantAnalysis) => {
// Make sure it's the variant analysis we're interested in
if (variantAnalysisId === variantAnalysis.id) {
await this.updateView({
if (variantAnalysis.id === variantAnalysisId) {
// Update model editor view
await this.updateModelEditorView({
isPreparing: false,
variantAnalysis,
});
// Update model alerts view
await this.modelAlertsView?.updateVariantAnalysis(variantAnalysis);
}
},
),
);
this.push(
this.variantAnalysisManager.onRepoStatesUpdated(async (e) => {
if (
e.variantAnalysisId === variantAnalysisId &&
e.repoState.downloadStatus ===
VariantAnalysisScannedRepositoryDownloadStatus.Succeeded
) {
await this.readAnalysisResults(
variantAnalysisId,
e.repoState.repositoryId,
);
}
}),
);
this.push(
this.variantAnalysisManager.onRepoResultsLoaded(async (e) => {
if (e.variantAnalysisId === variantAnalysisId) {
await this.modelAlertsView?.updateRepoResults(e);
}
}),
);
}
private async readAnalysisResults(
variantAnalysisId: number,
repositoryId: number,
) {
const variantAnalysis =
this.variantAnalysisManager.tryGetVariantAnalysis(variantAnalysisId);
if (!variantAnalysis) {
void this.app.logger.log(
`Could not find variant analysis with id ${variantAnalysisId}`,
);
throw new Error(
"There was an error when trying to retrieve variant analysis information",
);
}
const repository = variantAnalysis.scannedRepos?.find(
(r) => r.repository.id === repositoryId,
);
if (!repository) {
void this.app.logger.log(
`Could not find repository with id ${repositoryId} in scanned repos`,
);
throw new Error(
"There was an error when trying to retrieve repository information",
);
}
// Trigger loading the results for the repository. This will trigger a
// onRepoResultsLoaded event that we'll process.
await this.variantAnalysisManager.loadResults(
variantAnalysisId,
repository.repository.fullName,
);
}
}

84
extensions/ql-vscode/src/model-editor/modeled-method.ts
View File

@@ -1,3 +1,4 @@
import { assertNever } from "../common/helpers-pure";
import type { MethodSignature } from "./method";
import type { ModelingStatus } from "./shared/modeling-status";
@@ -169,3 +170,86 @@ export function calculateNewProvenance(
return "manual";
}
}
export function createModeledMethodKey(modeledMethod: ModeledMethod): string {
const canonicalModeledMethod = canonicalizeModeledMethod(modeledMethod);
return JSON.stringify(
canonicalModeledMethod,
// This ensures the keys are always in the same order
Object.keys(canonicalModeledMethod).sort(),
);
}
/**
* This method will reset any properties which are not used for the specific type of modeled method.
*
* It will also set the `provenance` to `manual` since multiple modelings of the same method with a
* different provenance are not actually different.
*
* The returned canonical modeled method should only be used for comparisons. It should not be used
* for display purposes, saving the model, or any other purpose which requires the original modeled
* method to be preserved.
*
* @param modeledMethod The modeled method to canonicalize
*/
function canonicalizeModeledMethod(
modeledMethod: ModeledMethod,
): ModeledMethod {
const methodSignature: MethodSignature = {
endpointType: modeledMethod.endpointType,
signature: modeledMethod.signature,
packageName: modeledMethod.packageName,
typeName: modeledMethod.typeName,
methodName: modeledMethod.methodName,
methodParameters: modeledMethod.methodParameters,
};
switch (modeledMethod.type) {
case "none":
return {
...methodSignature,
type: "none",
};
case "source":
return {
...methodSignature,
type: "source",
output: modeledMethod.output,
kind: modeledMethod.kind,
provenance: "manual",
};
case "sink":
return {
...methodSignature,
type: "sink",
input: modeledMethod.input,
kind: modeledMethod.kind,
provenance: "manual",
};
case "summary":
return {
...methodSignature,
type: "summary",
input: modeledMethod.input,
output: modeledMethod.output,
kind: modeledMethod.kind,
provenance: "manual",
};
case "neutral":
return {
...methodSignature,
type: "neutral",
kind: modeledMethod.kind,
provenance: "manual",
};
case "type":
return {
...methodSignature,
type: "type",
relatedTypeName: modeledMethod.relatedTypeName,
path: modeledMethod.path,
};
default:
assertNever(modeledMethod);
}
}

41
extensions/ql-vscode/src/model-editor/modeling-events.ts
View File

@@ -2,7 +2,7 @@ import type { App } from "../common/app";
import { DisposableObject } from "../common/disposable-object";
import type { AppEvent, AppEventEmitter } from "../common/events";
import type { DatabaseItem } from "../databases/local-databases";
import type { Method, Usage } from "./method";
import type { Method, MethodSignature, Usage } from "./method";
import type { ModelEvaluationRun } from "./model-evaluation-run";
import type { ModeledMethod } from "./modeled-method";
import type { Mode } from "./shared/mode";
@@ -58,13 +58,22 @@ interface ModelEvaluationRunChangedEvent {
interface RevealInModelEditorEvent {
dbUri: string;
method: Method;
method: MethodSignature;
}
interface FocusModelEditorEvent {
dbUri: string;
}
interface FocusModelAlertsViewEvent {
dbUri: string;
}
interface RevealInModelAlertsViewEvent {
dbUri: string;
modeledMethod: ModeledMethod;
}
export class ModelingEvents extends DisposableObject {
public readonly onActiveDbChanged: AppEvent<void>;
public readonly onDbOpened: AppEvent<DatabaseItem>;
@@ -79,6 +88,8 @@ export class ModelingEvents extends DisposableObject {
public readonly onModelEvaluationRunChanged: AppEvent<ModelEvaluationRunChangedEvent>;
public readonly onRevealInModelEditor: AppEvent<RevealInModelEditorEvent>;
public readonly onFocusModelEditor: AppEvent<FocusModelEditorEvent>;
public readonly onFocusModelAlertsView: AppEvent<FocusModelAlertsViewEvent>;
public readonly onRevealInModelAlertsView: AppEvent<RevealInModelAlertsViewEvent>;
private readonly onActiveDbChangedEventEmitter: AppEventEmitter<void>;
private readonly onDbOpenedEventEmitter: AppEventEmitter<DatabaseItem>;
@@ -93,6 +104,8 @@ export class ModelingEvents extends DisposableObject {
private readonly onModelEvaluationRunChangedEventEmitter: AppEventEmitter<ModelEvaluationRunChangedEvent>;
private readonly onRevealInModelEditorEventEmitter: AppEventEmitter<RevealInModelEditorEvent>;
private readonly onFocusModelEditorEventEmitter: AppEventEmitter<FocusModelEditorEvent>;
private readonly onFocusModelAlertsViewEventEmitter: AppEventEmitter<FocusModelAlertsViewEvent>;
private readonly onRevealInModelAlertsViewEventEmitter: AppEventEmitter<RevealInModelAlertsViewEvent>;
constructor(app: App) {
super();
@@ -165,6 +178,17 @@ export class ModelingEvents extends DisposableObject {
app.createEventEmitter<FocusModelEditorEvent>(),
);
this.onFocusModelEditor = this.onFocusModelEditorEventEmitter.event;
this.onFocusModelAlertsViewEventEmitter = this.push(
app.createEventEmitter<FocusModelAlertsViewEvent>(),
);
this.onFocusModelAlertsView = this.onFocusModelAlertsViewEventEmitter.event;
this.onRevealInModelAlertsViewEventEmitter = this.push(
app.createEventEmitter<RevealInModelAlertsViewEvent>(),
);
this.onRevealInModelAlertsView =
this.onRevealInModelAlertsViewEventEmitter.event;
}
public fireActiveDbChangedEvent() {
@@ -274,7 +298,7 @@ export class ModelingEvents extends DisposableObject {
});
}
public fireRevealInModelEditorEvent(dbUri: string, method: Method) {
public fireRevealInModelEditorEvent(dbUri: string, method: MethodSignature) {
this.onRevealInModelEditorEventEmitter.fire({
dbUri,
method,
@@ -286,4 +310,15 @@ export class ModelingEvents extends DisposableObject {
dbUri,
});
}
public fireFocusModelAlertsViewEvent(dbUri: string) {
this.onFocusModelAlertsViewEventEmitter.fire({ dbUri });
}
public fireRevealInModelAlertsViewEvent(
dbUri: string,
modeledMethod: ModeledMethod,
) {
this.onRevealInModelAlertsViewEventEmitter.fire({ dbUri, modeledMethod });
}
}

22
extensions/ql-vscode/src/model-editor/modeling-store.ts
View File

@@ -20,6 +20,7 @@ interface InternalDbModelingState {
selectedMethod: Method | undefined;
selectedUsage: Usage | undefined;
modelEvaluationRun: ModelEvaluationRun | undefined;
isModelAlertsViewOpen: boolean;
}
export interface DbModelingState {
@@ -34,6 +35,7 @@ export interface DbModelingState {
readonly selectedMethod: Method | undefined;
readonly selectedUsage: Usage | undefined;
readonly modelEvaluationRun: ModelEvaluationRun | undefined;
readonly isModelAlertsViewOpen: boolean;
}
export interface SelectedMethodDetails {
@@ -71,6 +73,7 @@ export class ModelingStore extends DisposableObject {
selectedUsage: undefined,
inProgressMethods: new Set(),
modelEvaluationRun: undefined,
isModelAlertsViewOpen: false,
});
this.modelingEvents.fireDbOpenedEvent(databaseItem);
@@ -498,4 +501,23 @@ export class ModelingStore extends DisposableObject {
state.modelEvaluationRun,
);
}
public isModelAlertsViewOpen(dbItem: DatabaseItem): boolean {
return this.getState(dbItem).isModelAlertsViewOpen ?? false;
}
private changeIsModelAlertsViewOpen(
dbItem: DatabaseItem,
updateState: (state: InternalDbModelingState) => void,
) {
const state = this.getState(dbItem);
updateState(state);
}
public updateIsModelAlertsViewOpen(dbItem: DatabaseItem, isOpen: boolean) {
this.changeIsModelAlertsViewOpen(dbItem, (state) => {
state.isModelAlertsViewOpen = isOpen;
});
}
}

75
extensions/ql-vscode/src/model-editor/shared/model-alerts-filter-sort.ts Normal file
View File

@@ -0,0 +1,75 @@
import type { ModelAlerts } from "../model-alerts/model-alerts";
export enum SortKey {
Alphabetically = "alphabetically",
NumberOfResults = "numberOfResults",
}
export type ModelAlertsFilterSortState = {
modelSearchValue: string;
repositorySearchValue: string;
sortKey: SortKey;
};
export const defaultFilterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "",
repositorySearchValue: "",
sortKey: SortKey.NumberOfResults,
};
export function filterAndSort(
modelAlerts: ModelAlerts[],
filterSortState: ModelAlertsFilterSortState,
): ModelAlerts[] {
if (!modelAlerts || modelAlerts.length === 0) {
return [];
}
return modelAlerts
.filter((item) => matchesFilter(item, filterSortState))
.sort((a, b) => {
switch (filterSortState.sortKey) {
case SortKey.Alphabetically:
return a.model.signature.localeCompare(b.model.signature);
case SortKey.NumberOfResults:
return (b.alerts.length || 0) - (a.alerts.length || 0);
default:
return 0;
}
});
}
function matchesFilter(
item: ModelAlerts,
filterSortState: ModelAlertsFilterSortState | undefined,
): boolean {
if (!filterSortState) {
return true;
}
return (
matchesRepository(item, filterSortState.repositorySearchValue) &&
matchesModel(item, filterSortState.modelSearchValue)
);
}
function matchesRepository(
item: ModelAlerts,
repositorySearchValue: string,
): boolean {
// We may want to only return alerts that have a repository match
// but for now just return true if the model has any alerts
// with a matching repo.
return item.alerts.some((alert) =>
alert.repository.fullName
.toLowerCase()
.includes(repositorySearchValue.toLowerCase()),
);
}
function matchesModel(item: ModelAlerts, modelSearchValue: string): boolean {
return item.model.signature
.toLowerCase()
.includes(modelSearchValue.toLowerCase());
}

84
extensions/ql-vscode/src/model-editor/shared/validation.ts
View File

@@ -1,6 +1,5 @@
import { createModeledMethodKey } from "../modeled-method";
import type { ModeledMethod, NeutralModeledMethod } from "../modeled-method";
import type { MethodSignature } from "../method";
import { assertNever } from "../../common/helpers-pure";
export type ModeledMethodValidationError = {
title: string;
@@ -9,80 +8,6 @@ export type ModeledMethodValidationError = {
index: number;
};
/**
* This method will reset any properties which are not used for the specific type of modeled method.
*
* It will also set the `provenance` to `manual` since multiple modelings of the same method with a
* different provenance are not actually different.
*
* The returned canonical modeled method should only be used for comparisons. It should not be used
* for display purposes, saving the model, or any other purpose which requires the original modeled
* method to be preserved.
*
* @param modeledMethod The modeled method to canonicalize
*/
function canonicalizeModeledMethod(
modeledMethod: ModeledMethod,
): ModeledMethod {
const methodSignature: MethodSignature = {
endpointType: modeledMethod.endpointType,
signature: modeledMethod.signature,
packageName: modeledMethod.packageName,
typeName: modeledMethod.typeName,
methodName: modeledMethod.methodName,
methodParameters: modeledMethod.methodParameters,
};
switch (modeledMethod.type) {
case "none":
return {
...methodSignature,
type: "none",
};
case "source":
return {
...methodSignature,
type: "source",
output: modeledMethod.output,
kind: modeledMethod.kind,
provenance: "manual",
};
case "sink":
return {
...methodSignature,
type: "sink",
input: modeledMethod.input,
kind: modeledMethod.kind,
provenance: "manual",
};
case "summary":
return {
...methodSignature,
type: "summary",
input: modeledMethod.input,
output: modeledMethod.output,
kind: modeledMethod.kind,
provenance: "manual",
};
case "neutral":
return {
...methodSignature,
type: "neutral",
kind: modeledMethod.kind,
provenance: "manual",
};
case "type":
return {
...methodSignature,
type: "type",
relatedTypeName: modeledMethod.relatedTypeName,
path: modeledMethod.path,
};
default:
assertNever(modeledMethod);
}
}
export function validateModeledMethods(
modeledMethods: ModeledMethod[],
): ModeledMethodValidationError[] {
@@ -97,12 +22,7 @@ export function validateModeledMethods(
// an error for any duplicates.
const seenModeledMethods = new Set<string>();
for (const modeledMethod of consideredModeledMethods) {
const canonicalModeledMethod = canonicalizeModeledMethod(modeledMethod);
const key = JSON.stringify(
canonicalModeledMethod,
// This ensures the keys are always in the same order
Object.keys(canonicalModeledMethod).sort(),
);
const key = createModeledMethodKey(modeledMethod);
if (seenModeledMethods.has(key)) {
result.push({

4
extensions/ql-vscode/src/model-editor/shared/view-state.ts
View File

@@ -19,3 +19,7 @@ export interface MethodModelingPanelViewState {
language: QueryLanguage | undefined;
modelConfig: ModelConfig;
}
export interface ModelAlertsViewState {
title: string;
}

3
extensions/ql-vscode/src/packaging/qlpack-file.schema.json
View File

@@ -111,6 +111,9 @@
"description": {
"type": "string"
},
"import": {
"type": "string"
},
"from": {
"type": "string"
}

1
extensions/ql-vscode/src/packaging/suite-instruction.ts
View File

@@ -8,5 +8,6 @@ export interface SuiteInstruction {
include?: Record<string, string[]>;
exclude?: Record<string, string[]>;
description?: string;
import?: string;
from?: string;
}

17
extensions/ql-vscode/src/query-server/query-runner.ts
View File

@@ -1,4 +1,4 @@
import { window } from "vscode";
import { window, Uri } from "vscode";
import type { CancellationToken, MessageItem } from "vscode";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import type { ProgressCallback } from "../common/vscode/progress";
@@ -63,9 +63,22 @@ export interface CoreQueryRun {
export type CoreCompletedQuery = CoreQueryResults &
Omit<CoreQueryRun, "evaluate">;
type OnQueryRunStartingListener = (dbPath: Uri) => Promise<void>;
export class QueryRunner {
constructor(public readonly qs: QueryServerClient) {}
// Event handlers that get notified whenever a query is about to start running.
// Can't use vscode EventEmitters since they are not asynchronous.
private readonly onQueryRunStartingListeners: OnQueryRunStartingListener[] =
[];
public onQueryRunStarting(listener: OnQueryRunStartingListener) {
this.onQueryRunStartingListeners.push(listener);
}
private async fireQueryRunStarting(dbPath: Uri) {
await Promise.all(this.onQueryRunStartingListeners.map((l) => l(dbPath)));
}
get cliServer(): CodeQLCliServer {
return this.qs.cliServer;
}
@@ -138,6 +151,8 @@ export class QueryRunner {
templates: Record<string, string> | undefined,
logger: BaseLogger,
): Promise<CoreQueryResults> {
await this.fireQueryRunStarting(Uri.file(dbPath));
return await compileAndRunQueryAgainstDatabaseCore(
this.qs,
dbPath,

14
extensions/ql-vscode/src/stories/Overview.stories.mdx → extensions/ql-vscode/src/stories/Overview.mdx
View File

@@ -1,4 +1,4 @@
import { Canvas, Meta, Story } from '@storybook/addon-docs';
import { Canvas, Meta, Story } from '@storybook/blocks';
import { VSCodeButton } from '@vscode/webview-ui-toolkit/react';
@@ -39,23 +39,29 @@ updated manually if new variables are added to VSCode. It can also be updated if
these variables, follow these steps:
1. Make sure you have selected the correct theme. If you want to use a variable which is currently not available and will be committed, please
select the **Dark+** theme. You can use **Preferences: Color Theme** in the *Command Palette* to select the theme.
select the **Dark+** theme. You can use **Preferences: Color Theme** in the *Command Palette* to select the theme.
2. Open a WebView in VSCode (for example the results of a query)
3. Open the *Command Palette* (Ctrl/Cmd+Shift+P)
4. Select **Developer: Open WebView Developer Tools**
5. Now, you will need to find the `<html>` element in the lowest-level `<iframe>`. See the image below:
<img src={iframeImage} alt="The iframe element showing in the VS Code webview developer tools element inspector" />
6. Once you have selected the `<html>` element as in the image above, click on **Show All Properties (... more)** (see image below). This will
expand all CSS variables.
expand all CSS variables.
<img src={stylesImage} alt="The styles tab of the VS Code webview developer tools element inspector" />
7. Copy all variables to the `src/stories/vscode-theme-dark.css` file.
8. Now, select the `<body>` element which is a direct child of the `<html>` element.
9. This time, you do not need to copy the variables. Instead, copy the styles on the `<body>` element to the `src/stories/vscode-theme-dark.css` file.
See the image below for which styles need to be copied.
See the image below for which styles need to be copied.
<img src={bodyImage} alt="The styles on the body element showing in the VS Code webview developer tools element inspector" />

29
extensions/ql-vscode/src/stories/common/SearchBox.stories.tsx Normal file
View File

@@ -0,0 +1,29 @@
import { useState } from "react";
import type { Meta } from "@storybook/react";
import { SearchBox as SearchBoxComponent } from "../../view/common/SearchBox";
export default {
title: "Search Box",
component: SearchBoxComponent,
argTypes: {
value: {
control: {
disable: true,
},
},
},
} as Meta<typeof SearchBoxComponent>;
export const SearchBox = () => {
const [value, setValue] = useState("");
return (
<SearchBoxComponent
value={value}
placeholder="Filter by x/y/z..."
onChange={setValue}
/>
);
};

108
extensions/ql-vscode/src/stories/model-alerts/ModelAlerts.stories.tsx Normal file
View File

@@ -0,0 +1,108 @@
import type { Meta, StoryFn } from "@storybook/react";
import { ModelAlerts as ModelAlertsComponent } from "../../view/model-alerts/ModelAlerts";
import { createMockVariantAnalysis } from "../../../test/factories/variant-analysis/shared/variant-analysis";
import { VariantAnalysisRepoStatus } from "../../variant-analysis/shared/variant-analysis";
import type { VariantAnalysisScannedRepositoryResult } from "../../variant-analysis/shared/variant-analysis";
import { createMockAnalysisAlert } from "../../../test/factories/variant-analysis/shared/analysis-alert";
export default {
title: "Model Alerts/Model Alerts",
component: ModelAlertsComponent,
} as Meta<typeof ModelAlertsComponent>;
const Template: StoryFn<typeof ModelAlertsComponent> = (args) => (
<ModelAlertsComponent {...args} />
);
const variantAnalysis = createMockVariantAnalysis({
modelPacks: [
{
name: "Model pack 1",
path: "/path/to/model-pack-1",
},
{
name: "Model pack 2",
path: "/path/to/model-pack-2",
},
],
scannedRepos: [
{
repository: {
id: 1,
fullName: "org/repo1",
private: false,
stargazersCount: 100,
updatedAt: new Date().toISOString(),
},
analysisStatus: VariantAnalysisRepoStatus.InProgress,
resultCount: 0,
artifactSizeInBytes: 0,
},
{
repository: {
id: 2,
fullName: "org/repo2",
private: false,
stargazersCount: 100,
updatedAt: new Date().toISOString(),
},
analysisStatus: VariantAnalysisRepoStatus.Succeeded,
resultCount: 0,
artifactSizeInBytes: 0,
},
{
repository: {
id: 3,
fullName: "org/repo3",
private: false,
stargazersCount: 100,
updatedAt: new Date().toISOString(),
},
analysisStatus: VariantAnalysisRepoStatus.Succeeded,
resultCount: 1,
artifactSizeInBytes: 0,
},
{
repository: {
id: 4,
fullName: "org/repo4",
private: false,
stargazersCount: 100,
updatedAt: new Date().toISOString(),
},
analysisStatus: VariantAnalysisRepoStatus.Succeeded,
resultCount: 3,
artifactSizeInBytes: 0,
},
],
});
const repoResults: VariantAnalysisScannedRepositoryResult[] = [
{
variantAnalysisId: variantAnalysis.id,
repositoryId: 2,
interpretedResults: [createMockAnalysisAlert(), createMockAnalysisAlert()],
},
{
variantAnalysisId: variantAnalysis.id,
repositoryId: 3,
interpretedResults: [
createMockAnalysisAlert(),
createMockAnalysisAlert(),
createMockAnalysisAlert(),
],
},
{
variantAnalysisId: variantAnalysis.id,
repositoryId: 4,
interpretedResults: [createMockAnalysisAlert()],
},
];
export const ModelAlerts = Template.bind({});
ModelAlerts.args = {
initialViewState: { title: "codeql/sql2o-models" },
variantAnalysis,
repoResults,
};

58
extensions/ql-vscode/src/stories/model-alerts/ModelAlertsHeader.stories.tsx Normal file
View File

@@ -0,0 +1,58 @@
import type { Meta, StoryFn } from "@storybook/react";
import { ModelAlertsHeader as ModelAlertsHeaderComponent } from "../../view/model-alerts/ModelAlertsHeader";
import { createMockVariantAnalysis } from "../../../test/factories/variant-analysis/shared/variant-analysis";
export default {
title: "Model Alerts/Model Alerts Header",
component: ModelAlertsHeaderComponent,
argTypes: {
openModelPackClick: {
action: "open-model-pack-clicked",
table: {
disable: true,
},
},
onViewLogsClick: {
action: "view-logs-clicked",
table: {
disable: true,
},
},
stopRunClick: {
action: "stop-run-clicked",
table: {
disable: true,
},
},
},
} as Meta<typeof ModelAlertsHeaderComponent>;
const Template: StoryFn<typeof ModelAlertsHeaderComponent> = (args) => (
<ModelAlertsHeaderComponent {...args} />
);
export const ModelAlertsHeader = Template.bind({});
ModelAlertsHeader.args = {
viewState: { title: "codeql/sql2o-models" },
variantAnalysis: createMockVariantAnalysis({
modelPacks: [
{
name: "Model pack 1",
path: "/path/to/model-pack-1",
},
{
name: "Model pack 2",
path: "/path/to/model-pack-2",
},
{
name: "Model pack 3",
path: "/path/to/model-pack-3",
},
{
name: "Model pack 4",
path: "/path/to/model-pack-4",
},
],
}),
};

30
extensions/ql-vscode/src/stories/model-alerts/ModelAlertsResults.stories.tsx Normal file
View File

@@ -0,0 +1,30 @@
import type { Meta, StoryFn } from "@storybook/react";
import { ModelAlertsResults as ModelAlertsResultsComponent } from "../../view/model-alerts/ModelAlertsResults";
import { createSinkModeledMethod } from "../../../test/factories/model-editor/modeled-method-factories";
import { createMockAnalysisAlert } from "../../../test/factories/variant-analysis/shared/analysis-alert";
export default {
title: "Model Alerts/Model Alerts Results",
component: ModelAlertsResultsComponent,
} as Meta<typeof ModelAlertsResultsComponent>;
const Template: StoryFn<typeof ModelAlertsResultsComponent> = (args) => (
<ModelAlertsResultsComponent {...args} />
);
export const ModelAlertsResults = Template.bind({});
ModelAlertsResults.args = {
modelAlerts: {
model: createSinkModeledMethod(),
alerts: [
{
repository: {
id: 1,
fullName: "expressjs/express",
},
alert: createMockAnalysisAlert(),
},
],
},
};

34
extensions/ql-vscode/src/stories/model-alerts/ModelPacks.stories.tsx Normal file
View File

@@ -0,0 +1,34 @@
import type { Meta, StoryFn } from "@storybook/react";
import { ModelPacks as ModelPacksComponent } from "../../view/model-alerts/ModelPacks";
export default {
title: "Model Alerts/Model Packs",
component: ModelPacksComponent,
argTypes: {
openModelPackClick: {
action: "open-model-pack-clicked",
table: {
disable: true,
},
},
},
} as Meta<typeof ModelPacksComponent>;
const Template: StoryFn<typeof ModelPacksComponent> = (args) => (
<ModelPacksComponent {...args} />
);
export const ModelPacks = Template.bind({});
ModelPacks.args = {
modelPacks: [
{
name: "Model pack 1",
path: "/path/to/model-pack-1",
},
{
name: "Model pack 2",
path: "/path/to/model-pack-2",
},
],
};

31
extensions/ql-vscode/src/stories/variant-analysis/AnalysisAlertResult.stories.tsx
View File

@@ -2,6 +2,7 @@ import type { Meta, StoryFn } from "@storybook/react";
import AnalysisAlertResult from "../../view/variant-analysis/AnalysisAlertResult";
import type { AnalysisAlert } from "../../variant-analysis/shared/analysis-result";
import { createMockAnalysisAlert } from "../../../test/factories/variant-analysis/shared/analysis-alert";
export default {
title: "Variant Analysis/Analysis Alert Result",
@@ -14,35 +15,7 @@ const Template: StoryFn<typeof AnalysisAlertResult> = (args) => (
export const Warning = Template.bind({});
const warningAlert: AnalysisAlert = {
message: {
tokens: [
{
t: "text",
text: "This is an empty block.",
},
],
},
shortDescription: "This is an empty block.",
fileLink: {
fileLinkPrefix:
"https://github.com/expressjs/express/blob/33e8dc303af9277f8a7e4f46abfdcb5e72f6797b",
filePath: "test/app.options.js",
},
severity: "Warning",
codeSnippet: {
startLine: 10,
endLine: 14,
text: " app.del('/', function(){});\n app.get('/users', function(req, res){});\n app.put('/users', function(req, res){});\n\n request(app)\n",
},
highlightedRegion: {
startLine: 12,
startColumn: 41,
endLine: 12,
endColumn: 43,
},
codeFlows: [],
};
const warningAlert: AnalysisAlert = createMockAnalysisAlert();
Warning.args = {
alert: warningAlert,

23
extensions/ql-vscode/src/stories/variant-analysis/RepositoriesSearch.stories.tsx
View File

@@ -1,23 +0,0 @@
import { useState } from "react";
import type { Meta } from "@storybook/react";
import { RepositoriesSearch as RepositoriesSearchComponent } from "../../view/variant-analysis/RepositoriesSearch";
export default {
title: "Variant Analysis/Repositories Search",
component: RepositoriesSearchComponent,
argTypes: {
value: {
control: {
disable: true,
},
},
},
} as Meta<typeof RepositoriesSearchComponent>;
export const RepositoriesSearch = () => {
const [value, setValue] = useState("");
return <RepositoriesSearchComponent value={value} onChange={setValue} />;
};

73
extensions/ql-vscode/src/variant-analysis/code-scanning-pack.ts
View File

@@ -1,20 +1,26 @@
import { join } from "path";
import { outputFile } from "fs-extra";
import { dump } from "js-yaml";
import { file } from "tmp-promise";
import type { BaseLogger } from "../common/logging";
import type { QueryLanguage } from "../common/query-language";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import type { QlPackDetails } from "./ql-pack-details";
import { getQlPackFilePath } from "../common/ql";
import { isSarifResultsQueryKind } from "../common/query-metadata";
import type { SuiteInstruction } from "../packaging/suite-instruction";
import { SARIF_RESULTS_QUERY_KINDS } from "../common/query-metadata";
import type { CancellationToken } from "vscode";
export async function resolveCodeScanningQueryPack(
logger: BaseLogger,
cliServer: CodeQLCliServer,
language: QueryLanguage,
token: CancellationToken,
): Promise<QlPackDetails> {
// Get pack
void logger.log(`Downloading pack for language: ${language}`);
const packName = `codeql/${language}-queries`;
const packDownloadResult = await cliServer.packDownload([packName]);
const packDownloadResult = await cliServer.packDownload([packName], token);
const downloadedPack = packDownloadResult.packs[0];
const packDir = join(
@@ -25,20 +31,40 @@ export async function resolveCodeScanningQueryPack(
// Resolve queries
void logger.log(`Resolving queries for pack: ${packName}`);
const suitePath = join(
packDir,
"codeql-suites",
`${language}-code-scanning.qls`,
);
const resolvedQueries = await cliServer.resolveQueries(suitePath);
const problemQueries = await filterToOnlyProblemQueries(
logger,
cliServer,
resolvedQueries,
);
const suiteYaml: SuiteInstruction[] = [
{
import: `codeql-suites/${language}-code-scanning.qls`,
from: `${downloadedPack.name}@${downloadedPack.version}`,
},
{
// This is necessary to ensure that the next import filter
// is applied correctly
exclude: {},
},
{
// Only include problem queries
include: {
kind: SARIF_RESULTS_QUERY_KINDS,
},
},
];
if (problemQueries.length === 0) {
let resolvedQueries: string[];
const suiteFile = await file({
postfix: ".qls",
});
const suitePath = suiteFile.path;
try {
await outputFile(suitePath, dump(suiteYaml), "utf8");
resolvedQueries = await cliServer.resolveQueries(suitePath);
} finally {
await suiteFile.cleanup();
}
if (resolvedQueries.length === 0) {
throw Error(
`No problem queries found in published query pack: ${packName}.`,
);
@@ -48,7 +74,7 @@ export async function resolveCodeScanningQueryPack(
const qlPackFilePath = await getQlPackFilePath(packDir);
const qlPackDetails: QlPackDetails = {
queryFiles: problemQueries,
queryFiles: resolvedQueries,
qlPackRootPath: packDir,
qlPackFilePath,
language,
@@ -56,20 +82,3 @@ export async function resolveCodeScanningQueryPack(
return qlPackDetails;
}
async function filterToOnlyProblemQueries(
logger: BaseLogger,
cliServer: CodeQLCliServer,
queries: string[],
): Promise<string[]> {
const problemQueries: string[] = [];
for (const query of queries) {
const queryMetadata = await cliServer.resolveMetadata(query);
if (isSarifResultsQueryKind(queryMetadata.kind)) {
problemQueries.push(query);
} else {
void logger.log(`Skipping non-problem query ${query}`);
}
}
return problemQueries;
}

42
extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
View File

@@ -37,24 +37,31 @@ import {
import type { QlPackFile } from "../packaging/qlpack-file";
import { expandShortPaths } from "../common/short-paths";
import type { QlPackDetails } from "./ql-pack-details";
import type { ModelPackDetails } from "../common/model-pack-details";
/**
* Well-known names for the query pack used by the server.
*/
const QUERY_PACK_NAME = "codeql-remote/query";
interface GeneratedQlPackDetails {
base64Pack: string;
modelPacks: ModelPackDetails[];
}
/**
* Two possibilities:
* 1. There is no qlpack.yml (or codeql-pack.yml) in this directory. Assume this is a lone query and generate a synthetic qlpack for it.
* 2. There is a qlpack.yml (or codeql-pack.yml) in this directory. Assume this is a query pack and use the yml to pack the query before uploading it.
*
* @returns the entire qlpack as a base64 string.
* @returns details about the generated QL pack.
*/
async function generateQueryPack(
cliServer: CodeQLCliServer,
qlPackDetails: QlPackDetails,
tmpDir: RemoteQueryTempDir,
): Promise<string> {
token: CancellationToken,
): Promise<GeneratedQlPackDetails> {
const workspaceFolders = getOnDiskWorkspaceFolders();
const extensionPacks = await getExtensionPacksToInject(
cliServer,
@@ -128,7 +135,7 @@ async function generateQueryPack(
...queryOpts,
// We need to specify the extension packs as dependencies so that they are included in the MRVA pack.
// The version range doesn't matter, since they'll always be found by source lookup.
...extensionPacks.map((p) => `--extension-pack=${p}@*`),
...extensionPacks.map((p) => `--extension-pack=${p.name}@*`),
];
} else {
precompilationOpts = ["--qlx"];
@@ -148,9 +155,13 @@ async function generateQueryPack(
bundlePath,
tmpDir.compiledPackDir,
precompilationOpts,
token,
);
const base64Pack = (await readFile(bundlePath)).toString("base64");
return base64Pack;
return {
base64Pack,
modelPacks: extensionPacks,
};
}
async function createNewQueryPack(
@@ -276,6 +287,7 @@ async function getPackedBundlePath(remoteQueryDir: string): Promise<string> {
interface PreparedRemoteQuery {
actionBranch: string;
base64Pack: string;
modelPacks: ModelPackDetails[];
repoSelection: RepositorySelection;
controllerRepo: Repository;
queryStartTime: number;
@@ -328,10 +340,15 @@ export async function prepareRemoteQueryRun(
const tempDir = await createRemoteQueriesTempDirectory();
let base64Pack: string;
let generatedPack: GeneratedQlPackDetails;
try {
base64Pack = await generateQueryPack(cliServer, qlPackDetails, tempDir);
generatedPack = await generateQueryPack(
cliServer,
qlPackDetails,
tempDir,
token,
);
} finally {
await tempDir.remoteQueryDir.cleanup();
}
@@ -351,7 +368,8 @@ export async function prepareRemoteQueryRun(
return {
actionBranch,
base64Pack,
base64Pack: generatedPack.base64Pack,
modelPacks: generatedPack.modelPacks,
repoSelection,
controllerRepo,
queryStartTime,
@@ -397,8 +415,8 @@ async function fixPackFile(
async function getExtensionPacksToInject(
cliServer: CodeQLCliServer,
workspaceFolders: string[],
): Promise<string[]> {
const result: string[] = [];
): Promise<ModelPackDetails[]> {
const result: ModelPackDetails[] = [];
if (cliServer.useExtensionPacks()) {
const extensionPacks = await cliServer.resolveQlpacks(
workspaceFolders,
@@ -415,7 +433,7 @@ async function getExtensionPacksToInject(
)}`,
);
}
result.push(name);
result.push({ name, path: paths[0] });
});
}
@@ -424,7 +442,7 @@ async function getExtensionPacksToInject(
async function addExtensionPacksAsDependencies(
queryPackDir: string,
extensionPacks: string[],
extensionPacks: ModelPackDetails[],
): Promise<void> {
const qlpackFile = await getQlPackFilePath(queryPackDir);
if (!qlpackFile) {
@@ -440,7 +458,7 @@ async function addExtensionPacksAsDependencies(
) as QlPackFile;
const dependencies = syntheticQueryPack.dependencies ?? {};
extensionPacks.forEach((name) => {
extensionPacks.forEach(({ name }) => {
// Add this extension pack as a dependency. It doesn't matter which
// version we specify, since we are guaranteed that the extension pack
// is resolved from source at the given path.

2
extensions/ql-vscode/src/variant-analysis/shared/variant-analysis.ts
View File

@@ -1,6 +1,7 @@
import type { Repository, RepositoryWithMetadata } from "./repository";
import type { AnalysisAlert, AnalysisRawResults } from "./analysis-result";
import { QueryLanguage } from "../../common/query-language";
import type { ModelPackDetails } from "../../common/model-pack-details";
export interface VariantAnalysis {
id: number;
@@ -13,6 +14,7 @@ export interface VariantAnalysis {
kind?: string;
};
queries?: VariantAnalysisQueries;
modelPacks?: ModelPackDetails[];
databases: {
repositories?: string[];
repositoryLists?: string[];

105
extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
View File

@@ -122,6 +122,21 @@ export class VariantAnalysisManager
public readonly onVariantAnalysisRemoved =
this._onVariantAnalysisRemoved.event;
private readonly _onRepoStateUpdated = this.push(
new EventEmitter<{
variantAnalysisId: number;
repoState: VariantAnalysisScannedRepositoryState;
}>(),
);
public readonly onRepoStatesUpdated = this._onRepoStateUpdated.event;
private readonly _onRepoResultsLoaded = this.push(
new EventEmitter<VariantAnalysisScannedRepositoryResult>(),
);
public readonly onRepoResultsLoaded = this._onRepoResultsLoaded.event;
private readonly variantAnalysisMonitor: VariantAnalysisMonitor;
private readonly variantAnalyses = new Map<number, VariantAnalysis>();
private readonly views = new Map<number, VariantAnalysisView>();
@@ -176,6 +191,8 @@ export class VariantAnalysisManager
"codeQL.monitorReauthenticatedVariantAnalysis":
this.monitorVariantAnalysis.bind(this),
"codeQL.openVariantAnalysisLogs": this.openVariantAnalysisLogs.bind(this),
"codeQLModelAlerts.openVariantAnalysisLogs":
this.openVariantAnalysisLogs.bind(this),
"codeQL.openVariantAnalysisView": this.showView.bind(this),
"codeQL.runVariantAnalysis":
this.runVariantAnalysisFromCommandPalette.bind(this),
@@ -221,42 +238,49 @@ export class VariantAnalysisManager
}
public async runVariantAnalysisFromPublishedPack(): Promise<void> {
return withProgress(async (progress, token) => {
progress({
maxStep: 7,
step: 0,
message: "Determining query language",
});
return withProgress(
async (progress, token) => {
progress({
maxStep: 7,
step: 0,
message: "Determining query language",
});
const language = await askForLanguage(this.cliServer);
if (!language) {
return;
}
const language = await askForLanguage(this.cliServer, true, token);
if (!language) {
return;
}
progress({
maxStep: 7,
step: 2,
message: "Downloading query pack and resolving queries",
});
progress({
maxStep: 7,
step: 2,
message: "Downloading query pack and resolving queries",
});
// Build up details to pass to the functions that run the variant analysis.
const qlPackDetails = await resolveCodeScanningQueryPack(
this.app.logger,
this.cliServer,
language,
);
// Build up details to pass to the functions that run the variant analysis.
const qlPackDetails = await resolveCodeScanningQueryPack(
this.app.logger,
this.cliServer,
language,
token,
);
await this.runVariantAnalysis(
qlPackDetails,
(p) =>
progress({
...p,
maxStep: p.maxStep + 3,
step: p.step + 3,
}),
token,
);
});
await this.runVariantAnalysis(
qlPackDetails,
(p) =>
progress({
...p,
maxStep: p.maxStep + 3,
step: p.step + 3,
}),
token,
);
},
{
title: "Run Variant Analysis",
cancellable: true,
},
);
}
private async runVariantAnalysisCommand(queryFiles: Uri[]): Promise<void> {
@@ -343,6 +367,7 @@ export class VariantAnalysisManager
const {
actionBranch,
base64Pack,
modelPacks,
repoSelection,
controllerRepo,
queryStartTime,
@@ -403,6 +428,7 @@ export class VariantAnalysisManager
const mappedVariantAnalysis = mapVariantAnalysisFromSubmission(
variantAnalysisSubmission,
variantAnalysisResponse,
modelPacks,
);
await this.onVariantAnalysisSubmitted(mappedVariantAnalysis);
@@ -609,6 +635,17 @@ export class VariantAnalysisManager
);
}
public getLoadedResultsForVariantAnalysis(variantAnalysisId: number) {
const variantAnalysis = this.variantAnalyses.get(variantAnalysisId);
if (!variantAnalysis) {
throw new Error(`No variant analysis with id: ${variantAnalysisId}`);
}
return this.variantAnalysisResultsManager.getLoadedResultsForVariantAnalysis(
variantAnalysis,
);
}
private async variantAnalysisRecordExists(
variantAnalysisId: number,
): Promise<boolean> {
@@ -676,6 +713,8 @@ export class VariantAnalysisManager
await this.getView(
repositoryResult.variantAnalysisId,
)?.sendRepositoryResults([repositoryResult]);
this._onRepoResultsLoaded.fire(repositoryResult);
}
private async onRepoStateUpdated(
@@ -691,6 +730,8 @@ export class VariantAnalysisManager
}
repoStates[repoState.repositoryId] = repoState;
this._onRepoStateUpdated.fire({ variantAnalysisId, repoState });
}
private async onDidChangeSessions(

11
extensions/ql-vscode/src/variant-analysis/variant-analysis-mapper.ts
View File

@@ -1,3 +1,4 @@
import type { ModelPackDetails } from "../common/model-pack-details";
import type {
VariantAnalysis as ApiVariantAnalysis,
VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository,
@@ -26,6 +27,7 @@ import {
export function mapVariantAnalysisFromSubmission(
submission: VariantAnalysisSubmission,
apiVariantAnalysis: ApiVariantAnalysis,
modelPacks: ModelPackDetails[],
): VariantAnalysis {
return mapVariantAnalysis(
{
@@ -37,6 +39,7 @@ export function mapVariantAnalysisFromSubmission(
kind: submission.query.kind,
},
queries: submission.queries,
modelPacks,
databases: submission.databases,
executionStartTime: submission.startTime,
},
@@ -59,7 +62,12 @@ export function mapUpdatedVariantAnalysis(
function mapVariantAnalysis(
currentVariantAnalysis: Pick<
VariantAnalysis,
"language" | "query" | "queries" | "databases" | "executionStartTime"
| "language"
| "query"
| "queries"
| "databases"
| "executionStartTime"
| "modelPacks"
>,
currentStatus: VariantAnalysisStatus | undefined,
response: ApiVariantAnalysis,
@@ -96,6 +104,7 @@ function mapVariantAnalysis(
language: currentVariantAnalysis.language,
query: currentVariantAnalysis.query,
queries: currentVariantAnalysis.queries,
modelPacks: currentVariantAnalysis.modelPacks,
databases: currentVariantAnalysis.databases,
executionStartTime: currentVariantAnalysis.executionStartTime,
createdAt: response.created_at,

23
extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts
View File

@@ -12,6 +12,7 @@ import { sarifParser } from "../common/sarif-parser";
import { extractAnalysisAlerts } from "./sarif-processing";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import { extractRawResults } from "./bqrs-processing";
import { VariantAnalysisRepoStatus } from "./shared/variant-analysis";
import type {
VariantAnalysis,
VariantAnalysisRepositoryTask,
@@ -305,6 +306,28 @@ export class VariantAnalysisResultsManager extends DisposableObject {
}
}
public getLoadedResultsForVariantAnalysis(
variantAnalysis: VariantAnalysis,
): VariantAnalysisScannedRepositoryResult[] {
const scannedRepos = variantAnalysis.scannedRepos?.filter(
(r) => r.analysisStatus === VariantAnalysisRepoStatus.Succeeded,
);
if (!scannedRepos) {
return [];
}
return scannedRepos
.map((scannedRepo) =>
this.cachedResults.get(
createCacheKey(variantAnalysis.id, scannedRepo.repository.fullName),
),
)
.filter(
(r): r is VariantAnalysisScannedRepositoryResult => r !== undefined,
);
}
public dispose(disposeHandler?: DisposeHandler) {
super.dispose(disposeHandler);

0
extensions/ql-vscode/src/view/variant-analysis/LinkIconButton.tsx → extensions/ql-vscode/src/view/common/LinkIconButton.tsx
View File

12
extensions/ql-vscode/src/view/variant-analysis/RepositoriesSearch.tsx → extensions/ql-vscode/src/view/common/SearchBox.tsx
View File

@@ -1,7 +1,7 @@
import { useCallback } from "react";
import { styled } from "styled-components";
import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react";
import { Codicon } from "../common";
import { Codicon } from "./icon";
const TextField = styled(VSCodeTextField)`
width: 100%;
@@ -9,12 +9,18 @@ const TextField = styled(VSCodeTextField)`
type Props = {
value: string;
placeholder: string;
onChange: (value: string) => void;
className?: string;
};
export const RepositoriesSearch = ({ value, onChange, className }: Props) => {
export const SearchBox = ({
value,
placeholder,
onChange,
className,
}: Props) => {
const handleInput = useCallback(
(e: InputEvent) => {
const target = e.target as HTMLInputElement;
@@ -26,7 +32,7 @@ export const RepositoriesSearch = ({ value, onChange, className }: Props) => {
return (
<TextField
placeholder="Filter by repository owner/name"
placeholder={placeholder}
value={value}
onInput={handleInput}
className={className}

154
extensions/ql-vscode/src/view/model-alerts/ModelAlerts.tsx
View File

@@ -1,10 +1,102 @@
import { useEffect } from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { styled } from "styled-components";
import { ModelAlertsHeader } from "./ModelAlertsHeader";
import type { ModelAlertsViewState } from "../../model-editor/shared/view-state";
import type { ToModelAlertsMessage } from "../../common/interface-types";
import type {
VariantAnalysis,
VariantAnalysisScannedRepositoryResult,
} from "../../variant-analysis/shared/variant-analysis";
import { vscode } from "../vscode-api";
import { ModelAlertsResults } from "./ModelAlertsResults";
import type { ModelAlerts } from "../../model-editor/model-alerts/model-alerts";
import { calculateModelAlerts } from "../../model-editor/model-alerts/alert-processor";
import { ModelAlertsSearchSortRow } from "./ModelAlertsSearchSortRow";
import {
defaultFilterSortState,
filterAndSort,
} from "../../model-editor/shared/model-alerts-filter-sort";
import type { ModelAlertsFilterSortState } from "../../model-editor/shared/model-alerts-filter-sort";
import type { ModeledMethod } from "../../model-editor/modeled-method";
type Props = {
initialViewState?: ModelAlertsViewState;
variantAnalysis?: VariantAnalysis;
repoResults?: VariantAnalysisScannedRepositoryResult[];
};
const SectionTitle = styled.h3`
font-size: medium;
font-weight: 500;
margin: 0;
padding-bottom: 10px;
`;
export function ModelAlerts({
initialViewState,
variantAnalysis: initialVariantAnalysis,
repoResults: initialRepoResults = [],
}: Props): React.JSX.Element {
const onOpenModelPackClick = useCallback((path: string) => {
vscode.postMessage({
t: "openModelPack",
path,
});
}, []);
const onStopRunClick = useCallback(() => {
vscode.postMessage({
t: "stopEvaluationRun",
});
}, []);
const [viewState, setViewState] = useState<ModelAlertsViewState | undefined>(
initialViewState,
);
const [variantAnalysis, setVariantAnalysis] = useState<
VariantAnalysis | undefined
>(initialVariantAnalysis);
const [repoResults, setRepoResults] =
useState<VariantAnalysisScannedRepositoryResult[]>(initialRepoResults);
const [filterSortValue, setFilterSortValue] =
useState<ModelAlertsFilterSortState>(defaultFilterSortState);
const [revealedModel, setRevealedModel] = useState<ModeledMethod | null>(
null,
);
export function ModelAlerts(): React.JSX.Element {
useEffect(() => {
const listener = (evt: MessageEvent) => {
if (evt.origin === window.origin) {
// TODO: handle messages
const msg: ToModelAlertsMessage = evt.data;
switch (msg.t) {
case "setModelAlertsViewState": {
setViewState(msg.viewState);
break;
}
case "setVariantAnalysis": {
setVariantAnalysis(msg.variantAnalysis);
break;
}
case "setRepoResults": {
setRepoResults((oldRepoResults) => {
const newRepoIds = msg.repoResults.map((r) => r.repositoryId);
return [
...oldRepoResults.filter(
(v) => !newRepoIds.includes(v.repositoryId),
),
...msg.repoResults,
];
});
break;
}
case "revealModel": {
setRevealedModel(msg.modeledMethod);
break;
}
}
} else {
// sanitize origin
const origin = evt.origin.replace(/\n|\r/g, "");
@@ -18,5 +110,59 @@ export function ModelAlerts(): React.JSX.Element {
};
}, []);
return <>hello world</>;
const modelAlerts = useMemo(() => {
if (!repoResults || !variantAnalysis) {
return [];
}
const modelAlerts = calculateModelAlerts(variantAnalysis, repoResults);
return filterAndSort(modelAlerts, filterSortValue);
}, [filterSortValue, variantAnalysis, repoResults]);
if (viewState === undefined || variantAnalysis === undefined) {
return <></>;
}
const openLogs = () => {
vscode.postMessage({
t: "openActionsLogs",
variantAnalysisId: variantAnalysis.id,
});
};
const onViewLogsClick =
variantAnalysis.actionsWorkflowRunId === undefined ? undefined : openLogs;
return (
<>
<ModelAlertsHeader
viewState={viewState}
variantAnalysis={variantAnalysis}
openModelPackClick={onOpenModelPackClick}
onViewLogsClick={onViewLogsClick}
stopRunClick={onStopRunClick}
></ModelAlertsHeader>
<div>
<SectionTitle>Model alerts</SectionTitle>
<ModelAlertsSearchSortRow
filterSortValue={filterSortValue}
onFilterSortChange={setFilterSortValue}
/>
<div>
{modelAlerts.map((alerts, i) => (
// We're using the index as the key here which is not recommended.
// but we don't have a unique identifier for models. In the future,
// we may need to consider coming up with unique identifiers for models
// and using those as keys.
<ModelAlertsResults
key={i}
modelAlerts={alerts}
revealedModel={revealedModel}
/>
))}
</div>
</div>
</>
);
}

39
extensions/ql-vscode/src/view/model-alerts/ModelAlertsActions.tsx Normal file
View File

@@ -0,0 +1,39 @@
import { styled } from "styled-components";
import { VSCodeButton } from "@vscode/webview-ui-toolkit/react";
import { VariantAnalysisStatus } from "../../variant-analysis/shared/variant-analysis";
type ModelAlertsActionsProps = {
variantAnalysisStatus: VariantAnalysisStatus;
onStopRunClick: () => void;
};
const Container = styled.div`
margin-left: auto;
display: flex;
gap: 1em;
`;
const Button = styled(VSCodeButton)`
white-space: nowrap;
`;
export const ModelAlertsActions = ({
variantAnalysisStatus,
onStopRunClick,
}: ModelAlertsActionsProps) => {
return (
<Container>
{variantAnalysisStatus === VariantAnalysisStatus.InProgress && (
<Button appearance="secondary" onClick={onStopRunClick}>
Stop evaluation
</Button>
)}
{variantAnalysisStatus === VariantAnalysisStatus.Canceling && (
<Button appearance="secondary" disabled={true}>
Stopping evaluation
</Button>
)}
</Container>
);
};

94
extensions/ql-vscode/src/view/model-alerts/ModelAlertsHeader.tsx Normal file
View File

@@ -0,0 +1,94 @@
import { useMemo } from "react";
import { parseDate } from "../../common/date";
import { styled } from "styled-components";
import type { ModelAlertsViewState } from "../../model-editor/shared/view-state";
import {
getSkippedRepoCount,
getTotalResultCount,
hasRepoScanCompleted,
isRepoScanSuccessful,
} from "../../variant-analysis/shared/variant-analysis";
import type { VariantAnalysis } from "../../variant-analysis/shared/variant-analysis";
import { ViewTitle } from "../common";
import { ModelAlertsActions } from "./ModelAlertsActions";
import { ModelPacks } from "./ModelPacks";
import { VariantAnalysisStats } from "../variant-analysis/VariantAnalysisStats";
type Props = {
viewState: ModelAlertsViewState;
variantAnalysis: VariantAnalysis;
openModelPackClick: (path: string) => void;
onViewLogsClick?: () => void;
stopRunClick: () => void;
};
const Container = styled.div`
display: flex;
flex-direction: column;
`;
const Row = styled.div`
display: flex;
align-items: flex-start;
`;
export const ModelAlertsHeader = ({
viewState,
variantAnalysis,
openModelPackClick,
onViewLogsClick,
stopRunClick,
}: Props) => {
const totalScannedRepositoryCount = useMemo(() => {
return variantAnalysis.scannedRepos?.length ?? 0;
}, [variantAnalysis.scannedRepos]);
const completedRepositoryCount = useMemo(() => {
return (
variantAnalysis.scannedRepos?.filter((repo) => hasRepoScanCompleted(repo))
?.length ?? 0
);
}, [variantAnalysis.scannedRepos]);
const successfulRepositoryCount = useMemo(() => {
return (
variantAnalysis.scannedRepos?.filter((repo) => isRepoScanSuccessful(repo))
?.length ?? 0
);
}, [variantAnalysis.scannedRepos]);
const resultCount = useMemo(() => {
return getTotalResultCount(variantAnalysis.scannedRepos);
}, [variantAnalysis.scannedRepos]);
const skippedRepositoryCount = useMemo(() => {
return getSkippedRepoCount(variantAnalysis.skippedRepos);
}, [variantAnalysis.skippedRepos]);
return (
<>
<Container>
<Row>
<ViewTitle>Model evaluation results for {viewState.title}</ViewTitle>
</Row>
<Row>
<ModelPacks
modelPacks={variantAnalysis.modelPacks || []}
openModelPackClick={openModelPackClick}
></ModelPacks>
<ModelAlertsActions
variantAnalysisStatus={variantAnalysis.status}
onStopRunClick={stopRunClick}
/>
</Row>
<VariantAnalysisStats
variantAnalysisStatus={variantAnalysis.status}
totalRepositoryCount={totalScannedRepositoryCount}
completedRepositoryCount={completedRepositoryCount}
successfulRepositoryCount={successfulRepositoryCount}
skippedRepositoryCount={skippedRepositoryCount}
resultCount={resultCount}
createdAt={parseDate(variantAnalysis.createdAt)}
completedAt={parseDate(variantAnalysis.completedAt)}
onViewLogsClick={onViewLogsClick}
/>
</Container>
</>
);
};

133
extensions/ql-vscode/src/view/model-alerts/ModelAlertsResults.tsx Normal file
View File

@@ -0,0 +1,133 @@
import { styled } from "styled-components";
import type { ModelAlerts } from "../../model-editor/model-alerts/model-alerts";
import { Codicon } from "../common";
import { VSCodeBadge, VSCodeLink } from "@vscode/webview-ui-toolkit/react";
import { useCallback, useEffect, useRef, useState } from "react";
import { formatDecimal } from "../../common/number";
import AnalysisAlertResult from "../variant-analysis/AnalysisAlertResult";
import { MethodName } from "../model-editor/MethodName";
import { ModelDetails } from "./ModelDetails";
import { vscode } from "../vscode-api";
import { createModeledMethodKey } from "../../model-editor/modeled-method";
import type { ModeledMethod } from "../../model-editor/modeled-method";
// This will ensure that these icons have a className which we can use in the TitleContainer
const ExpandCollapseCodicon = styled(Codicon)``;
const TitleContainer = styled.button`
display: flex;
gap: 0.5em;
align-items: center;
width: 100%;
color: var(--vscode-editor-foreground);
background-color: transparent;
border: none;
cursor: pointer;
&:disabled {
cursor: default;
${ExpandCollapseCodicon} {
color: var(--vscode-disabledForeground);
}
}
`;
const ModelTypeText = styled.span`
font-size: 0.85em;
color: var(--vscode-descriptionForeground);
`;
const ViewLink = styled(VSCodeLink)`
white-space: nowrap;
padding: 0 0 0.25em 1em;
`;
const ModelDetailsContainer = styled.div`
padding-top: 10px;
`;
const AlertsContainer = styled.ul`
list-style-type: none;
margin: 1em 0 0;
padding: 0.5em 0 0 0;
`;
const Alert = styled.li`
margin-bottom: 1em;
background-color: var(--vscode-notifications-background);
`;
interface Props {
modelAlerts: ModelAlerts;
revealedModel: ModeledMethod | null;
}
export const ModelAlertsResults = ({
modelAlerts,
revealedModel,
}: Props): React.JSX.Element => {
const [isExpanded, setExpanded] = useState(true);
const viewInModelEditor = useCallback(
() =>
vscode.postMessage({
t: "revealInModelEditor",
method: modelAlerts.model,
}),
[modelAlerts.model],
);
const ref = useRef<HTMLElement>();
useEffect(() => {
if (
revealedModel &&
createModeledMethodKey(modelAlerts.model) ===
createModeledMethodKey(revealedModel)
) {
ref.current?.scrollIntoView({
behavior: "smooth",
block: "center",
});
}
}, [modelAlerts.model, revealedModel]);
return (
<div>
<TitleContainer onClick={() => setExpanded(!isExpanded)}>
{isExpanded && (
<ExpandCollapseCodicon name="chevron-down" label="Collapse" />
)}
{!isExpanded && (
<ExpandCollapseCodicon name="chevron-right" label="Expand" />
)}
<VSCodeBadge>{formatDecimal(modelAlerts.alerts.length)}</VSCodeBadge>
<MethodName {...modelAlerts.model}></MethodName>
<ModelTypeText>{modelAlerts.model.type}</ModelTypeText>
<ViewLink
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
viewInModelEditor();
}}
>
View
</ViewLink>
</TitleContainer>
{isExpanded && (
<>
<ModelDetailsContainer>
<ModelDetails model={modelAlerts.model} />
</ModelDetailsContainer>
<AlertsContainer>
{modelAlerts.alerts.map((r, i) => (
<Alert key={i}>
<AnalysisAlertResult alert={r.alert} />
</Alert>
))}
</AlertsContainer>
</>
)}
</div>
);
};

87
extensions/ql-vscode/src/view/model-alerts/ModelAlertsSearchSortRow.tsx Normal file
View File

@@ -0,0 +1,87 @@
import { useCallback } from "react";
import type { Dispatch, SetStateAction } from "react";
import { styled } from "styled-components";
import type {
ModelAlertsFilterSortState,
SortKey,
} from "../../model-editor/shared/model-alerts-filter-sort";
import { SearchBox } from "../common/SearchBox";
import { ModelAlertsSort } from "./ModelAlertsSort";
type Props = {
filterSortValue: ModelAlertsFilterSortState;
onFilterSortChange: Dispatch<SetStateAction<ModelAlertsFilterSortState>>;
};
const Container = styled.div`
display: flex;
gap: 1em;
width: 100%;
margin-bottom: 1em;
`;
const ModelsSearchColumn = styled(SearchBox)`
flex: 2;
`;
const RepositoriesSearchColumn = styled(SearchBox)`
flex: 2;
`;
const SortColumn = styled(ModelAlertsSort)`
flex: 1;
`;
export const ModelAlertsSearchSortRow = ({
filterSortValue,
onFilterSortChange,
}: Props) => {
const handleModelSearchValueChange = useCallback(
(searchValue: string) => {
onFilterSortChange((oldValue) => ({
...oldValue,
modelSearchValue: searchValue,
}));
},
[onFilterSortChange],
);
const handleRepositorySearchValueChange = useCallback(
(searchValue: string) => {
onFilterSortChange((oldValue) => ({
...oldValue,
repositorySearchValue: searchValue,
}));
},
[onFilterSortChange],
);
const handleSortKeyChange = useCallback(
(sortKey: SortKey) => {
onFilterSortChange((oldValue) => ({
...oldValue,
sortKey,
}));
},
[onFilterSortChange],
);
return (
<Container>
<ModelsSearchColumn
placeholder="Filter by model"
value={filterSortValue.modelSearchValue}
onChange={handleModelSearchValueChange}
/>
<RepositoriesSearchColumn
placeholder="Filter by repository owner/name"
value={filterSortValue.repositorySearchValue}
onChange={handleRepositorySearchValueChange}
/>
<SortColumn
value={filterSortValue.sortKey}
onChange={handleSortKeyChange}
/>
</Container>
);
};

37
extensions/ql-vscode/src/view/model-alerts/ModelAlertsSort.tsx Normal file
View File

@@ -0,0 +1,37 @@
import { useCallback } from "react";
import { styled } from "styled-components";
import { VSCodeDropdown, VSCodeOption } from "@vscode/webview-ui-toolkit/react";
import { SortKey } from "../../model-editor/shared/model-alerts-filter-sort";
import { Codicon } from "../common";
const Dropdown = styled(VSCodeDropdown)`
width: 100%;
`;
type Props = {
value: SortKey;
onChange: (value: SortKey) => void;
className?: string;
};
export const ModelAlertsSort = ({ value, onChange, className }: Props) => {
const handleInput = useCallback(
(e: InputEvent) => {
const target = e.target as HTMLSelectElement;
onChange(target.value as SortKey);
},
[onChange],
);
return (
<Dropdown value={value} onInput={handleInput} className={className}>
<Codicon name="sort-precedence" label="Sort..." slot="indicator" />
<VSCodeOption value={SortKey.Alphabetically}>Alphabetically</VSCodeOption>
<VSCodeOption value={SortKey.NumberOfResults}>
Number of results
</VSCodeOption>
</Dropdown>
);
};

52
extensions/ql-vscode/src/view/model-alerts/ModelDetails.tsx Normal file
View File

@@ -0,0 +1,52 @@
import { styled } from "styled-components";
import {
modeledMethodSupportsInput,
modeledMethodSupportsKind,
modeledMethodSupportsOutput,
} from "../../model-editor/modeled-method";
import type { ModeledMethod } from "../../model-editor/modeled-method";
const DetailsContainer = styled.div`
display: flex;
`;
const Detail = styled.span`
display: flex;
margin-right: 30px;
`;
const Label = styled.span`
color: var(--vscode-descriptionForeground);
margin-right: 10px;
`;
const Value = styled.span``;
export const ModelDetails = ({ model }: { model: ModeledMethod }) => {
return (
<DetailsContainer>
<Detail>
<Label>Model type:</Label>
<Value>{model.type}</Value>
</Detail>
{modeledMethodSupportsInput(model) && (
<Detail>
<Label>Input:</Label>
<Value>{model.input}</Value>
</Detail>
)}
{modeledMethodSupportsOutput(model) && (
<Detail>
<Label>Output:</Label>
<Value>{model.output}</Value>
</Detail>
)}
{modeledMethodSupportsKind(model) && (
<Detail>
<Label>Kind:</Label>
<Value>{model.kind}</Value>
</Detail>
)}
</DetailsContainer>
);
};

47
extensions/ql-vscode/src/view/model-alerts/ModelPacks.tsx Normal file
View File

@@ -0,0 +1,47 @@
import { styled } from "styled-components";
import { LinkIconButton } from "../common/LinkIconButton";
import type { ModelPackDetails } from "../../common/model-pack-details";
const Container = styled.div`
display: block;
`;
const Title = styled.h3`
font-size: medium;
font-weight: 500;
margin: 0;
`;
const List = styled.ul`
list-style: none;
padding: 0;
margin-top: 5px;
`;
export const ModelPacks = ({
modelPacks,
openModelPackClick,
}: {
modelPacks: ModelPackDetails[];
openModelPackClick: (path: string) => void;
}): React.JSX.Element => {
if (modelPacks.length <= 0) {
return <></>;
}
return (
<Container>
<Title>Model packs</Title>
<List>
{modelPacks.map((modelPack) => (
<li key={modelPack.path}>
<LinkIconButton onClick={() => openModelPackClick(modelPack.path)}>
<span slot="start" className="codicon codicon-file-code"></span>
{modelPack.name}
</LinkIconButton>
</li>
))}
</List>
</Container>
);
};

4
extensions/ql-vscode/src/view/model-editor/LibraryRow.tsx
View File

@@ -15,6 +15,7 @@ import {
import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
import type { AccessPathSuggestionOptions } from "../../model-editor/suggestions";
import { getCandidates } from "../../model-editor/shared/auto-model-candidates";
import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
const LibraryContainer = styled.div`
background-color: var(--vscode-peekViewResult-background);
@@ -80,6 +81,7 @@ export type LibraryRowProps = {
hideModeledMethods: boolean;
revealedMethodSignature: string | null;
accessPathSuggestions?: AccessPathSuggestionOptions;
evaluationRun: ModelEvaluationRunState | undefined;
onChange: (methodSignature: string, modeledMethods: ModeledMethod[]) => void;
onMethodClick: (methodSignature: string) => void;
onSaveModelClick: (methodSignatures: string[]) => void;
@@ -105,6 +107,7 @@ export const LibraryRow = ({
hideModeledMethods,
revealedMethodSignature,
accessPathSuggestions,
evaluationRun,
onChange,
onMethodClick,
onSaveModelClick,
@@ -260,6 +263,7 @@ export const LibraryRow = ({
hideModeledMethods={hideModeledMethods}
revealedMethodSignature={revealedMethodSignature}
accessPathSuggestions={accessPathSuggestions}
evaluationRun={evaluationRun}
onChange={onChange}
onMethodClick={onMethodClick}
/>

37
extensions/ql-vscode/src/view/model-editor/MethodName.tsx
View File

@@ -1,33 +1,48 @@
import { styled } from "styled-components";
import type { Method } from "../../model-editor/method";
const Name = styled.span`
font-family: var(--vscode-editor-font-family);
word-break: break-all;
`;
const TypeMethodName = (method: Method) => {
if (!method.typeName) {
return <>{method.methodName}</>;
const TypeMethodName = ({
typeName,
methodName,
}: {
typeName?: string;
methodName?: string;
}) => {
if (!typeName) {
return <>{methodName}</>;
}
if (!method.methodName) {
return <>{method.typeName}</>;
if (!methodName) {
return <>{typeName}</>;
}
return (
<>
{method.typeName}.{method.methodName}
{typeName}.{methodName}
</>
);
};
export const MethodName = (method: Method): React.JSX.Element => {
export const MethodName = ({
packageName,
typeName,
methodName,
methodParameters,
}: {
packageName: string;
typeName?: string;
methodName?: string;
methodParameters?: string;
}): React.JSX.Element => {
return (
<Name>
{method.packageName && <>{method.packageName}.</>}
<TypeMethodName {...method} />
{method.methodParameters}
{packageName && <>{packageName}.</>}
<TypeMethodName typeName={typeName} methodName={methodName} />
{methodParameters}
</Name>
);
};

74
extensions/ql-vscode/src/view/model-editor/MethodRow.tsx
View File

@@ -1,4 +1,5 @@
import {
VSCodeBadge,
VSCodeButton,
VSCodeLink,
VSCodeProgressRing,
@@ -38,6 +39,8 @@ import type { AccessPathOption } from "../../model-editor/suggestions";
import { ModelInputSuggestBox } from "./ModelInputSuggestBox";
import { ModelOutputSuggestBox } from "./ModelOutputSuggestBox";
import { getModelsAsDataLanguage } from "../../model-editor/languages";
import { ModelAlertsIndicator } from "./ModelAlertsIndicator";
import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
const ApiOrMethodRow = styled.div`
min-height: calc(var(--input-height) * 1px);
@@ -47,11 +50,15 @@ const ApiOrMethodRow = styled.div`
gap: 0.5em;
`;
const UsagesButton = styled.button`
color: var(--vscode-editor-foreground);
background-color: var(--vscode-input-background);
border: none;
border-radius: 40%;
const ModelButtonsContainer = styled.div`
min-height: calc(var(--input-height) * 1px);
display: flex;
flex-direction: row;
align-items: center;
gap: 1em;
`;
const UsagesButton = styled(VSCodeBadge)`
cursor: pointer;
`;
@@ -82,6 +89,7 @@ export type MethodRowProps = {
revealedMethodSignature: string | null;
inputAccessPathSuggestions?: AccessPathOption[];
outputAccessPathSuggestions?: AccessPathOption[];
evaluationRun: ModelEvaluationRunState | undefined;
onChange: (methodSignature: string, modeledMethods: ModeledMethod[]) => void;
onMethodClick: (methodSignature: string) => void;
};
@@ -119,6 +127,7 @@ const ModelableMethodRow = forwardRef<HTMLElement | undefined, MethodRowProps>(
revealedMethodSignature,
inputAccessPathSuggestions,
outputAccessPathSuggestions,
evaluationRun,
onChange,
onMethodClick,
} = props;
@@ -349,30 +358,37 @@ const ModelableMethodRow = forwardRef<HTMLElement | undefined, MethodRowProps>(
/>
</DataGridCell>
<DataGridCell>
{index === 0 ? (
<CodiconRow
appearance="icon"
aria-label="Add new model"
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
handleAddModelClick();
}}
disabled={addModelButtonDisabled}
>
<Codicon name="add" />
</CodiconRow>
) : (
<CodiconRow
appearance="icon"
aria-label="Remove model"
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
removeModelClickedHandlers[index]();
}}
>
<Codicon name="trash" />
</CodiconRow>
)}
<ModelButtonsContainer>
<ModelAlertsIndicator
viewState={viewState}
modeledMethod={modeledMethod}
evaluationRun={evaluationRun}
></ModelAlertsIndicator>
{index === 0 ? (
<CodiconRow
appearance="icon"
aria-label="Add new model"
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
handleAddModelClick();
}}
disabled={addModelButtonDisabled}
>
<Codicon name="add" />
</CodiconRow>
) : (
<CodiconRow
appearance="icon"
aria-label="Remove model"
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
removeModelClickedHandlers[index]();
}}
>
<Codicon name="trash" />
</CodiconRow>
)}
</ModelButtonsContainer>
</DataGridCell>
</DataGridRow>
);

54
extensions/ql-vscode/src/view/model-editor/ModelAlertsIndicator.tsx Normal file
View File

@@ -0,0 +1,54 @@
import { styled } from "styled-components";
import type { ModeledMethod } from "../../model-editor/modeled-method";
import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
import { VSCodeBadge } from "@vscode/webview-ui-toolkit/react";
import { vscode } from "../vscode-api";
const ModelAlertsButton = styled(VSCodeBadge)`
cursor: pointer;
`;
export type Props = {
viewState: ModelEditorViewState;
modeledMethod: ModeledMethod;
evaluationRun: ModelEvaluationRunState | undefined;
};
export const ModelAlertsIndicator = ({
viewState,
modeledMethod,
evaluationRun,
}: Props) => {
if (!viewState.showEvaluationUi) {
return null;
}
if (!evaluationRun?.variantAnalysis || !modeledMethod) {
return null;
}
const revealInModelAlertsView = () => {
vscode.postMessage({
t: "revealInModelAlertsView",
modeledMethod,
});
};
// TODO: Once we have alert provenance, we can show actual alert counts here.
// For now, we show a random number.
const number = Math.floor(Math.random() * 10);
return (
<ModelAlertsButton
role="button"
aria-label="Model alerts"
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
revealInModelAlertsView();
}}
>
{number}
</ModelAlertsButton>
);
};

3
extensions/ql-vscode/src/view/model-editor/ModelEditor.tsx
View File

@@ -11,7 +11,7 @@ import type { ModeledMethod } from "../../model-editor/modeled-method";
import { assertNever } from "../../common/helpers-pure";
import { vscode } from "../vscode-api";
import { calculateModeledPercentage } from "../../model-editor/shared/modeled-percentage";
import { LinkIconButton } from "../variant-analysis/LinkIconButton";
import { LinkIconButton } from "../common/LinkIconButton";
import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
import { ModeledMethodsList } from "./ModeledMethodsList";
import { percentFormatter } from "./formatters";
@@ -436,6 +436,7 @@ export function ModelEditor({
hideModeledMethods={hideModeledMethods}
revealedMethodSignature={revealedMethodSignature}
accessPathSuggestions={accessPathSuggestions}
evaluationRun={evaluationRun}
onChange={onChange}
onMethodClick={onMethodClick}
onSaveModelClick={onSaveModelClick}

15
extensions/ql-vscode/src/view/model-editor/ModelEvaluation.tsx
View File

@@ -1,10 +1,11 @@
import { styled } from "styled-components";
import { VSCodeButton, VSCodeLink } from "@vscode/webview-ui-toolkit/react";
import type { ModeledMethod } from "../../model-editor/modeled-method";
import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
import { modelEvaluationRunIsRunning } from "../../model-editor/shared/model-evaluation-run-state";
import { ModelEditorProgressRing } from "./ModelEditorProgressRing";
import { LinkIconButton } from "../variant-analysis/LinkIconButton";
import { LinkIconButton } from "../common/LinkIconButton";
export type Props = {
viewState: ModelEditorViewState;
@@ -16,6 +17,11 @@ export type Props = {
evaluationRun: ModelEvaluationRunState | undefined;
};
const RunLink = styled(VSCodeLink)`
display: flex;
align-items: center;
`;
export const ModelEvaluation = ({
viewState,
modeledMethods,
@@ -34,7 +40,8 @@ export const ModelEvaluation = ({
const shouldShowStopButton = !shouldShowEvaluateButton;
const shouldShowEvaluationRunLink = !!evaluationRun;
const shouldShowEvaluationRunLink =
!!evaluationRun && evaluationRun.variantAnalysis;
const customModelsExist = Object.values(modeledMethods).some(
(methods) => methods.filter((m) => m.type !== "none").length > 0,
@@ -60,12 +67,12 @@ export const ModelEvaluation = ({
</VSCodeButton>
)}
{shouldShowEvaluationRunLink && (
<VSCodeLink>
<RunLink>
<LinkIconButton onClick={openModelAlertsView}>
<span slot="end" className="codicon codicon-link-external"></span>
Evaluation run
</LinkIconButton>
</VSCodeLink>
</RunLink>
)}
</>
);

4
extensions/ql-vscode/src/view/model-editor/ModeledMethodDataGrid.tsx
View File

@@ -8,6 +8,7 @@ import type { ModelEditorViewState } from "../../model-editor/shared/view-state"
import { ScreenReaderOnly } from "../common/ScreenReaderOnly";
import { DataGrid, DataGridCell } from "../common/DataGrid";
import type { AccessPathSuggestionOptions } from "../../model-editor/suggestions";
import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
export const MULTIPLE_MODELS_GRID_TEMPLATE_COLUMNS =
"0.5fr 0.125fr 0.125fr 0.125fr 0.125fr max-content";
@@ -23,6 +24,7 @@ export type ModeledMethodDataGridProps = {
hideModeledMethods: boolean;
revealedMethodSignature: string | null;
accessPathSuggestions?: AccessPathSuggestionOptions;
evaluationRun: ModelEvaluationRunState | undefined;
onChange: (methodSignature: string, modeledMethods: ModeledMethod[]) => void;
onMethodClick: (methodSignature: string) => void;
};
@@ -38,6 +40,7 @@ export const ModeledMethodDataGrid = ({
hideModeledMethods,
revealedMethodSignature,
accessPathSuggestions,
evaluationRun,
onChange,
onMethodClick,
}: ModeledMethodDataGridProps) => {
@@ -101,6 +104,7 @@ export const ModeledMethodDataGrid = ({
revealedMethodSignature={revealedMethodSignature}
inputAccessPathSuggestions={inputAccessPathSuggestions}
outputAccessPathSuggestions={outputAccessPathSuggestions}
evaluationRun={evaluationRun}
onChange={onChange}
onMethodClick={onMethodClick}
/>

4
extensions/ql-vscode/src/view/model-editor/ModeledMethodsList.tsx
View File

@@ -9,6 +9,7 @@ import {
} from "../../model-editor/shared/sorting";
import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
import type { AccessPathSuggestionOptions } from "../../model-editor/suggestions";
import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
export type ModeledMethodsListProps = {
methods: Method[];
@@ -19,6 +20,7 @@ export type ModeledMethodsListProps = {
processedByAutoModelMethods: Set<string>;
revealedMethodSignature: string | null;
accessPathSuggestions?: AccessPathSuggestionOptions;
evaluationRun: ModelEvaluationRunState | undefined;
viewState: ModelEditorViewState;
hideModeledMethods: boolean;
onChange: (methodSignature: string, modeledMethods: ModeledMethod[]) => void;
@@ -48,6 +50,7 @@ export const ModeledMethodsList = ({
hideModeledMethods,
revealedMethodSignature,
accessPathSuggestions,
evaluationRun,
onChange,
onMethodClick,
onSaveModelClick,
@@ -98,6 +101,7 @@ export const ModeledMethodsList = ({
hideModeledMethods={hideModeledMethods}
revealedMethodSignature={revealedMethodSignature}
accessPathSuggestions={accessPathSuggestions}
evaluationRun={evaluationRun}
onChange={onChange}
onMethodClick={onMethodClick}
onSaveModelClick={onSaveModelClick}

1
extensions/ql-vscode/src/view/model-editor/__tests__/LibraryRow.spec.tsx
View File

@@ -37,6 +37,7 @@ describe(LibraryRow.name, () => {
selectedSignatures={new Set()}
inProgressMethods={new Set()}
processedByAutoModelMethods={new Set()}
evaluationRun={undefined}
viewState={viewState}
hideModeledMethods={false}
revealedMethodSignature={null}

1
extensions/ql-vscode/src/view/model-editor/__tests__/MethodRow.spec.tsx
View File

@@ -45,6 +45,7 @@ describe(MethodRow.name, () => {
modelingInProgress={false}
processedByAutoModel={false}
revealedMethodSignature={null}
evaluationRun={undefined}
viewState={viewState}
onChange={onChange}
onMethodClick={onMethodClick}

70
extensions/ql-vscode/src/view/model-editor/__tests__/ModelAlertsIndicator.spec.tsx Normal file
View File

@@ -0,0 +1,70 @@
import { render as reactRender, screen } from "@testing-library/react";
import { createMethod } from "../../../../test/factories/model-editor/method-factories";
import { createSummaryModeledMethod } from "../../../../test/factories/model-editor/modeled-method-factories";
import { createMockModelEditorViewState } from "../../../../test/factories/model-editor/view-state";
import type { Props } from "../ModelAlertsIndicator";
import { ModelAlertsIndicator } from "../ModelAlertsIndicator";
import { createMockVariantAnalysis } from "../../../../test/factories/variant-analysis/shared/variant-analysis";
import { VariantAnalysisStatus } from "../../../variant-analysis/shared/variant-analysis";
describe(ModelAlertsIndicator.name, () => {
const method = createMethod();
const modeledMethod = createSummaryModeledMethod(method);
const evaluationRun = {
isPreparing: false,
variantAnalysis: createMockVariantAnalysis({
status: VariantAnalysisStatus.Succeeded,
}),
};
const render = (props: Partial<Props> = {}) =>
reactRender(
<ModelAlertsIndicator
viewState={createMockModelEditorViewState({ showEvaluationUi: true })}
modeledMethod={modeledMethod}
evaluationRun={evaluationRun}
{...props}
/>,
);
describe("when showEvaluationUi is false", () => {
it("does not render anything", () => {
render({
viewState: createMockModelEditorViewState({ showEvaluationUi: false }),
});
expect(screen.queryByRole("button")).not.toBeInTheDocument();
});
});
describe("when there is no evaluation run", () => {
it("does not render anything", () => {
render({
evaluationRun: undefined,
});
expect(screen.queryByRole("button")).not.toBeInTheDocument();
});
});
describe("when there is no modeled method", () => {
it("does not render anything", () => {
render({
modeledMethod: undefined,
});
expect(screen.queryByRole("button")).not.toBeInTheDocument();
});
});
describe("when there is an evaluation run and a modeled method", () => {
// TODO: Once we have alert provenance, this will be an actual alert count instead of a random number.
it("renders a button with a random number", () => {
render();
const button = screen.queryByRole("button");
expect(button).toBeInTheDocument();
expect(button).toHaveTextContent(/\d/);
});
});
});

23
extensions/ql-vscode/src/view/model-editor/__tests__/ModelEvaluation.spec.tsx
View File

@@ -98,7 +98,7 @@ describe(ModelEvaluation.name, () => {
expect(screen.queryByText("Stop evaluation")).not.toBeInTheDocument();
});
it("renders 'Stop evaluation' button and 'Evaluation run' link when there is an in progress evaluation", () => {
it("renders 'Stop evaluation' button when there is an in progress evaluation, but no variant analysis yet", () => {
render({
evaluationRun: {
isPreparing: true,
@@ -112,6 +112,27 @@ describe(ModelEvaluation.name, () => {
stopEvaluationButton?.getElementsByTagName("input")[0],
).toBeEnabled();
expect(screen.queryByText("Evaluation run")).not.toBeInTheDocument();
expect(screen.queryByText("Evaluate")).not.toBeInTheDocument();
});
it("renders 'Stop evaluation' button and 'Evaluation run' link when there is an in progress evaluation with variant analysis", () => {
render({
evaluationRun: {
isPreparing: false,
variantAnalysis: createMockVariantAnalysis({
status: VariantAnalysisStatus.InProgress,
}),
},
});
const stopEvaluationButton = screen.queryByText("Stop evaluation");
expect(stopEvaluationButton).toBeInTheDocument();
expect(
stopEvaluationButton?.getElementsByTagName("input")[0],
).toBeEnabled();
expect(screen.queryByText("Evaluation run")).toBeInTheDocument();
expect(screen.queryByText("Evaluate")).not.toBeInTheDocument();

1
extensions/ql-vscode/src/view/model-editor/__tests__/ModeledMethodDataGrid.spec.tsx
View File

@@ -59,6 +59,7 @@ describe(ModeledMethodDataGrid.name, () => {
selectedSignatures={new Set()}
inProgressMethods={new Set()}
processedByAutoModelMethods={new Set()}
evaluationRun={undefined}
viewState={viewState}
hideModeledMethods={false}
revealedMethodSignature={null}

1
extensions/ql-vscode/src/view/model-editor/__tests__/ModeledMethodsList.spec.tsx
View File

@@ -60,6 +60,7 @@ describe(ModeledMethodsList.name, () => {
selectedSignatures={new Set()}
inProgressMethods={new Set()}
processedByAutoModelMethods={new Set()}
evaluationRun={undefined}
viewState={viewState}
hideModeledMethods={false}
revealedMethodSignature={null}

2
extensions/ql-vscode/src/view/variant-analysis/QueryDetails.tsx
View File

@@ -1,6 +1,6 @@
import { styled } from "styled-components";
import { ViewTitle } from "../common";
import { LinkIconButton } from "./LinkIconButton";
import { LinkIconButton } from "../common/LinkIconButton";
export type QueryDetailsProps = {
queryName: string;

5
extensions/ql-vscode/src/view/variant-analysis/RepositoriesSearchSortRow.tsx
View File

@@ -6,7 +6,7 @@ import type {
RepositoriesFilterSortState,
SortKey,
} from "../../variant-analysis/shared/variant-analysis-filter-sort";
import { RepositoriesSearch } from "./RepositoriesSearch";
import { SearchBox } from "../common/SearchBox";
import { RepositoriesSort } from "./RepositoriesSort";
import { RepositoriesFilter } from "./RepositoriesFilter";
import { RepositoriesResultFormat } from "./RepositoriesResultFormat";
@@ -29,7 +29,7 @@ const Container = styled.div`
margin-bottom: 1em;
`;
const RepositoriesSearchColumn = styled(RepositoriesSearch)`
const RepositoriesSearchColumn = styled(SearchBox)`
flex: 3;
`;
@@ -99,6 +99,7 @@ export const RepositoriesSearchSortRow = ({
<Container>
<RepositoriesSearchColumn
value={filterSortValue.searchValue}
placeholder="Filter by repository owner/name"
onChange={handleSearchValueChange}
/>
<RepositoriesFilterColumn

4
extensions/ql-vscode/src/view/vscode-api.ts
View File

@@ -1,6 +1,7 @@
import type {
FromCompareViewMessage,
FromMethodModelingMessage,
FromModelAlertsMessage,
FromModelEditorMessage,
FromResultsViewMsg,
FromVariantAnalysisMessage,
@@ -17,7 +18,8 @@ export interface VsCodeApi {
| FromCompareViewMessage
| FromVariantAnalysisMessage
| FromModelEditorMessage
| FromMethodModelingMessage,
| FromMethodModelingMessage
| FromModelAlertsMessage,
): void;
/**

2
extensions/ql-vscode/supported_cli_versions.json
View File

@@ -1,5 +1,5 @@
[
"v2.16.4",
"v2.16.6",
"v2.15.5",
"v2.14.6",
"v2.13.5",

33
extensions/ql-vscode/test/factories/variant-analysis/shared/analysis-alert.ts Normal file
View File

@@ -0,0 +1,33 @@
import type { AnalysisAlert } from "../../../../src/variant-analysis/shared/analysis-result";
export function createMockAnalysisAlert(): AnalysisAlert {
return {
message: {
tokens: [
{
t: "text",
text: "This is an empty block.",
},
],
},
shortDescription: "This is an empty block.",
fileLink: {
fileLinkPrefix:
"https://github.com/expressjs/express/blob/33e8dc303af9277f8a7e4f46abfdcb5e72f6797b",
filePath: "test/app.options.js",
},
severity: "Warning",
codeSnippet: {
startLine: 10,
endLine: 14,
text: " app.del('/', function(){});\n app.get('/users', function(req, res){});\n app.put('/users', function(req, res){});\n\n request(app)\n",
},
highlightedRegion: {
startLine: 12,
startColumn: 41,
endLine: 12,
endColumn: 43,
},
codeFlows: [],
};
}

4
extensions/ql-vscode/test/factories/variant-analysis/shared/variant-analysis.ts
View File

@@ -9,6 +9,7 @@ import { createMockScannedRepos } from "./scanned-repositories";
import { createMockSkippedRepos } from "./skipped-repositories";
import { createMockRepository } from "./repository";
import { QueryLanguage } from "../../../../src/common/query-language";
import type { ModelPackDetails } from "../../../../src/common/model-pack-details";
export function createMockVariantAnalysis({
status = VariantAnalysisStatus.InProgress,
@@ -16,12 +17,14 @@ export function createMockVariantAnalysis({
skippedRepos = createMockSkippedRepos(),
executionStartTime = faker.number.int(),
language = QueryLanguage.Javascript,
modelPacks = undefined,
}: {
status?: VariantAnalysisStatus;
scannedRepos?: VariantAnalysisScannedRepository[];
skippedRepos?: VariantAnalysisSkippedRepositories;
executionStartTime?: number | undefined;
language?: QueryLanguage;
modelPacks?: ModelPackDetails[] | undefined;
}): VariantAnalysis {
return {
id: faker.number.int(),
@@ -37,6 +40,7 @@ export function createMockVariantAnalysis({
filePath: "a-query-file-path",
text: "a-query-text",
},
modelPacks,
databases: {
repositories: ["1", "2", "3"],
},

130
extensions/ql-vscode/test/unit-tests/model-editor/modeled-method.test.ts
View File

@@ -1,6 +1,13 @@
import { createNoneModeledMethod } from "../../factories/model-editor/modeled-method-factories";
import {
createNeutralModeledMethod,
createNoneModeledMethod,
createSinkModeledMethod,
createSourceModeledMethod,
createSummaryModeledMethod,
} from "../../factories/model-editor/modeled-method-factories";
import type { ModeledMethod } from "../../../src/model-editor/modeled-method";
import {
createModeledMethodKey,
modeledMethodSupportsInput,
modeledMethodSupportsKind,
modeledMethodSupportsOutput,
@@ -50,3 +57,124 @@ describe("modeledMethodSupportsProvenance", () => {
}
});
});
describe("createModeledMethodKey", () => {
it("should create a key for a modeled method", () => {
const modeledMethod = createNoneModeledMethod();
const key = createModeledMethodKey(modeledMethod);
const expectedKey =
'{"endpointType":"method","methodName":"createQuery","methodParameters":"(String)","packageName":"org.sql2o","signature":"org.sql2o.Connection#createQuery(String)","type":"none","typeName":"Connection"}';
expect(key).toBe(expectedKey);
});
it("should check that two modeled methods are the same", () => {
const modeledMethod = createSummaryModeledMethod();
const key = createModeledMethodKey(modeledMethod);
const modeledMethod2 = createSummaryModeledMethod();
const key2 = createModeledMethodKey(modeledMethod2);
// Object references are different, but the keys are the same.
expect(modeledMethod).not.toBe(modeledMethod2);
expect(key).toEqual(key2);
});
it("should always set provenance to manual", () => {
const modeledMethod = createSinkModeledMethod({
provenance: "df-generated",
});
const key = createModeledMethodKey(modeledMethod);
expect(key).not.toContain('"provenance":"df-generated"');
expect(key).toContain('"provenance":"manual"');
});
describe("ignores unused properties", () => {
it("for source modeled methods", () => {
const modeledMethod = createSourceModeledMethod({
output: "ReturnValue",
...{
input: "Argument[this]",
},
});
const key = createModeledMethodKey(modeledMethod);
const modeledMethod2 = createSourceModeledMethod({
output: "ReturnValue",
...{
input: "Argument[1]",
},
});
const key2 = createModeledMethodKey(modeledMethod2);
expect(key).not.toContain("input");
expect(key).toEqual(key2);
});
it("for sink modeled methods", () => {
const modeledMethod = createSinkModeledMethod({
input: "Argument[this]",
...{
output: "ReturnValue",
},
});
const key = createModeledMethodKey(modeledMethod);
const modeledMethod2 = createSinkModeledMethod({
input: "Argument[this]",
...{
output: "Argument[this]",
},
});
const key2 = createModeledMethodKey(modeledMethod2);
expect(key).not.toContain("output");
expect(key).toEqual(key2);
});
it("for summary modeled methods", () => {
const modeledMethod = createSummaryModeledMethod({
input: "Argument[this]",
output: "ReturnValue",
...{ supported: true },
});
const key = createModeledMethodKey(modeledMethod);
const modeledMethod2 = createSummaryModeledMethod({
input: "Argument[this]",
output: "ReturnValue",
...{ supported: false },
});
const key2 = createModeledMethodKey(modeledMethod2);
expect(key).not.toContain("supported");
expect(key).toEqual(key2);
});
it("for neutral modeled methods", () => {
const modeledMethod = createNeutralModeledMethod({
type: "neutral",
...{
input: "Argument[this]",
output: "ReturnValue",
},
});
const key = createModeledMethodKey(modeledMethod);
const modeledMethod2 = createNeutralModeledMethod({
type: "neutral",
...{
input: "Argument[1]",
output: "ReturnValue",
},
});
const key2 = createModeledMethodKey(modeledMethod2);
expect(key).not.toContain("input");
expect(key).not.toContain("output");
expect(key).toEqual(key2);
});
});
});

158
extensions/ql-vscode/test/unit-tests/model-editor/shared/model-alerts-filter-sort.test.ts Normal file
View File

@@ -0,0 +1,158 @@
import type { ModelAlerts } from "../../../../src/model-editor/model-alerts/model-alerts";
import type { ModelAlertsFilterSortState } from "../../../../src/model-editor/shared/model-alerts-filter-sort";
import {
SortKey,
filterAndSort,
} from "../../../../src/model-editor/shared/model-alerts-filter-sort";
import { createSinkModeledMethod } from "../../../factories/model-editor/modeled-method-factories";
import { createMockAnalysisAlert } from "../../../factories/variant-analysis/shared/analysis-alert";
import { shuffle } from "../../../vscode-tests/utils/list-helpers";
describe("model alerts filter sort", () => {
const modelAlerts: ModelAlerts[] = [
{
model: createSinkModeledMethod({
signature: "foo.m1",
}),
alerts: [
{
alert: createMockAnalysisAlert(),
repository: {
id: 1,
fullName: "r1",
},
},
{
alert: createMockAnalysisAlert(),
repository: {
id: 2,
fullName: "r2",
},
},
{
alert: createMockAnalysisAlert(),
repository: {
id: 3,
fullName: "r3",
},
},
{
alert: createMockAnalysisAlert(),
repository: {
id: 4,
fullName: "r4",
},
},
],
},
{
model: createSinkModeledMethod({
signature: "foo.m2",
}),
alerts: [
{
alert: createMockAnalysisAlert(),
repository: {
id: 1,
fullName: "r1",
},
},
{
alert: createMockAnalysisAlert(),
repository: {
id: 2,
fullName: "r2",
},
},
],
},
{
model: createSinkModeledMethod({
signature: "bar.m1",
}),
alerts: [
{
alert: createMockAnalysisAlert(),
repository: {
id: 1,
fullName: "r1",
},
},
],
},
];
it("should return an empty array if no model alerts", () => {
const filterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "",
repositorySearchValue: "",
sortKey: SortKey.Alphabetically,
};
const result = filterAndSort([], filterSortState);
expect(result).toEqual([]);
});
it("should filter model alerts based on the model search value", () => {
const filterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "m1",
repositorySearchValue: "",
sortKey: SortKey.Alphabetically,
};
const result = filterAndSort(modelAlerts, filterSortState);
expect(result.includes(modelAlerts[0])).toBeTruthy();
expect(result.includes(modelAlerts[2])).toBeTruthy();
});
it("should filter model alerts based on the repository search value", () => {
const filterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "",
repositorySearchValue: "r2",
sortKey: SortKey.Alphabetically,
};
const result = filterAndSort(modelAlerts, filterSortState);
expect(result.includes(modelAlerts[0])).toBeTruthy();
expect(result.includes(modelAlerts[1])).toBeTruthy();
});
it("should sort model alerts alphabetically", () => {
const filterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "",
repositorySearchValue: "",
sortKey: SortKey.Alphabetically,
};
const result = filterAndSort(shuffle([...modelAlerts]), filterSortState);
expect(result).toEqual([modelAlerts[2], modelAlerts[0], modelAlerts[1]]);
});
it("should sort model alerts by number of results", () => {
const filterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "",
repositorySearchValue: "",
sortKey: SortKey.NumberOfResults,
};
const result = filterAndSort(shuffle([...modelAlerts]), filterSortState);
expect(result).toEqual([modelAlerts[0], modelAlerts[1], modelAlerts[2]]);
});
it("should filter and sort model alerts", () => {
const filterSortState: ModelAlertsFilterSortState = {
modelSearchValue: "m1",
repositorySearchValue: "r1",
sortKey: SortKey.NumberOfResults,
};
const result = filterAndSort(shuffle([...modelAlerts]), filterSortState);
expect(result).toEqual([modelAlerts[0], modelAlerts[2]]);
});
});

2
extensions/ql-vscode/test/unit-tests/variant-analysis/variant-analysis-mapper.test.ts
View File

@@ -31,6 +31,7 @@ describe(mapVariantAnalysisFromSubmission.name, () => {
const result = mapVariantAnalysisFromSubmission(
mockSubmission,
mockApiResponse,
[],
);
const {
@@ -54,6 +55,7 @@ describe(mapVariantAnalysisFromSubmission.name, () => {
text: mockSubmission.query.text,
kind: "table",
},
modelPacks: [],
databases: {
repositories: ["1", "2", "3"],
},

4
extensions/ql-vscode/test/vscode-tests/activated-extension/model-editor/model-evaluator.test.ts
View File

@@ -7,6 +7,7 @@ import type { ModelEvaluationRun } from "../../../../src/model-editor/model-eval
import { ModelEvaluator } from "../../../../src/model-editor/model-evaluator";
import type { ModelingEvents } from "../../../../src/model-editor/modeling-events";
import type { ModelingStore } from "../../../../src/model-editor/modeling-store";
import type { ExtensionPack } from "../../../../src/model-editor/shared/extension-pack";
import type { VariantAnalysisManager } from "../../../../src/variant-analysis/variant-analysis-manager";
import { createMockLogger } from "../../../__mocks__/loggerMock";
import { createMockModelingEvents } from "../../../__mocks__/model-editor/modelingEventsMock";
@@ -23,6 +24,7 @@ describe("Model Evaluator", () => {
let variantAnalysisManager: VariantAnalysisManager;
let dbItem: DatabaseItem;
let language: QueryLanguage;
let extensionPack: ExtensionPack;
let updateView: jest.Mock;
let getModelEvaluationRunMock = jest.fn();
@@ -40,6 +42,7 @@ describe("Model Evaluator", () => {
});
dbItem = mockedObject<DatabaseItem>({});
language = QueryLanguage.Java;
extensionPack = mockedObject<ExtensionPack>({});
updateView = jest.fn();
modelEvaluator = new ModelEvaluator(
@@ -50,6 +53,7 @@ describe("Model Evaluator", () => {
variantAnalysisManager,
dbItem,
language,
extensionPack,
updateView,
);
});

47
extensions/ql-vscode/test/vscode-tests/cli-integration/databases/database-fetcher.test.ts
View File

@@ -10,8 +10,9 @@ import {
DB_URL,
getActivatedExtension,
storagePath,
testprojLoc,
} from "../../global.helper";
import { remove } from "fs-extra";
import { existsSync, remove, utimesSync } from "fs-extra";
import { createMockApp } from "../../../__mocks__/appMock";
/**
@@ -43,8 +44,8 @@ describe("database-fetcher", () => {
await remove(storagePath);
});
describe("importArchiveDatabase", () => {
it("should add a database from a folder", async () => {
describe("importLocalDatabase", () => {
it("should add a database from an archive", async () => {
const uri = Uri.file(dbLoc);
const databaseFetcher = new DatabaseFetcher(
createMockApp(),
@@ -52,7 +53,7 @@ describe("database-fetcher", () => {
storagePath,
cli,
);
let dbItem = await databaseFetcher.importArchiveDatabase(
let dbItem = await databaseFetcher.importLocalDatabase(
uri.toString(true),
progressCallback,
);
@@ -63,6 +64,44 @@ describe("database-fetcher", () => {
expect(dbItem.name).toBe("db");
expect(dbItem.databaseUri.fsPath).toBe(join(storagePath, "db", "db"));
});
it("should import a testproj database", async () => {
const databaseFetcher = new DatabaseFetcher(
createMockApp(),
databaseManager,
storagePath,
cli,
);
let dbItem = await databaseFetcher.importLocalDatabase(
Uri.file(testprojLoc).toString(true),
progressCallback,
);
expect(dbItem).toBe(databaseManager.currentDatabaseItem);
expect(dbItem).toBe(databaseManager.databaseItems[0]);
expect(dbItem).toBeDefined();
dbItem = dbItem!;
expect(dbItem.name).toBe("db");
expect(dbItem.databaseUri.fsPath).toBe(join(storagePath, "db"));
// Now that we have fetched it. Check for re-importing
// Delete a file in the imported database and we can check if the file is recreated
const srczip = join(dbItem.databaseUri.fsPath, "src.zip");
await remove(srczip);
// Attempt 1: re-import database should be a no-op since timestamp of imported database is newer
await databaseManager.maybeReimportTestDatabase(dbItem.databaseUri);
expect(existsSync(srczip)).toBeFalsy();
// Attempt 3: re-import database should re-import the database after updating modified time
utimesSync(
join(testprojLoc, "codeql-database.yml"),
new Date(),
new Date(),
);
await databaseManager.maybeReimportTestDatabase(dbItem.databaseUri, true);
expect(existsSync(srczip)).toBeTruthy();
});
});
describe("promptImportInternetDatabase", () => {

Some files were not shown because too many files have changed in this diff Show More