mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into robertbrignull/telemetry

Browse Source
This commit is contained in:
Robert
2023-10-05 15:19:07 +01:00
parent 4227ff6338 1806108166
commit d499ff3cf4
165 changed files with 6924 additions and 3236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
extensions/ql-vscode/CHANGELOG.md
View File

@@ -2,6 +2,18 @@
## [UNRELEASED]
- Fix a bug where the query to Find Definitions in database source files would not be cancelled appropriately. [#2885](https://github.com/github/vscode-codeql/pull/2885)
- It is now possible to show the language of query history items using the `%l` specifier in the `codeQL.queryHistory.format` setting. Note that this only works for queries run after this upgrade, and older items will show `unknown` as a language. [#2892](https://github.com/github/vscode-codeql/pull/2892)
- Increase the required version of VS Code to 1.82.0. [#2877](https://github.com/github/vscode-codeql/pull/2877)
- Fix a bug where the query server was restarted twice after configuration changes. [#2884](https://github.com/github/vscode-codeql/pull/2884).
## 1.9.1 - 29 September 2023
- Add warning when using a VS Code version older than 1.82.0. [#2854](https://github.com/github/vscode-codeql/pull/2854)
- Fix a bug when parsing large evaluation log summaries. [#2858](https://github.com/github/vscode-codeql/pull/2858)
- Right-align and format numbers in raw result tables. [#2864](https://github.com/github/vscode-codeql/pull/2864)
- Remove rate limit warning notifications when using Code Search to add repositories to a variant analysis list. [#2812](https://github.com/github/vscode-codeql/pull/2812)
## 1.9.0 - 19 September 2023
- Release the [CodeQL model editor](https://codeql.github.com/docs/codeql/codeql-for-visual-studio-code/using-the-codeql-model-editor) to create CodeQL model packs for Java frameworks. Open the editor using the "CodeQL: Open CodeQL Model Editor (Beta)" command. [#2823](https://github.com/github/vscode-codeql/pull/2823)

17
extensions/ql-vscode/media/dark/sort-alpha.svg
View File

@@ -1,17 +0,0 @@
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
<g>
<g>
<g>
<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72
250.027,338.24 432,338.24 432,304.32" fill="#C5C5C5"/>
<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
L113.28,140.48l41.387,110.507H71.893z" fill="#C5C5C5"/>
</g>
</g>
</g>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 953 B

3
extensions/ql-vscode/media/dark/sort-date.svg
View File

@@ -1,3 +0,0 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#C5C5C5"/>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 449 B

17
extensions/ql-vscode/media/light/sort-alpha.svg
View File

@@ -1,17 +0,0 @@
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
<g>
<g>
<g>
<polygon points="234.24,9.067 183.893,59.413 284.587,59.413 "/>
<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72
250.027,338.24 432,338.24 432,304.32 "/>
<polygon points="234.24,422.933 283.947,373.227 184.533,373.227 "/>
<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
L113.28,140.48l41.387,110.507H71.893z"/>
</g>
</g>
</g>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 894 B

3
extensions/ql-vscode/media/light/sort-date.svg
View File

@@ -1,3 +0,0 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#424242"/>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 449 B

1964
extensions/ql-vscode/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

301
extensions/ql-vscode/package.json
View File

@@ -4,7 +4,7 @@
"description": "CodeQL for Visual Studio Code",
"author": "GitHub",
"private": true,
"version": "1.9.1",
"version": "1.9.2",
"publisher": "GitHub",
"license": "MIT",
"icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,7 +13,7 @@
"url": "https://github.com/github/vscode-codeql"
},
"engines": {
"vscode": "^1.67.0",
"vscode": "^1.82.0",
"node": "^18.15.0",
"npm": ">=7.20.6"
},
@@ -760,6 +760,78 @@
"command": "codeQLDatabases.addDatabaseSource",
"title": "Add Database Source to Workspace"
},
{
"command": "codeQLDatabases.displayAllLanguages",
"title": "All languages"
},
{
"command": "codeQLDatabases.displayAllLanguagesSelected",
"title": "All languages (selected)"
},
{
"command": "codeQLDatabases.displayCpp",
"title": "C/C++"
},
{
"command": "codeQLDatabases.displayCppSelected",
"title": "C/C++ (selected)"
},
{
"command": "codeQLDatabases.displayCsharp",
"title": "C#"
},
{
"command": "codeQLDatabases.displayCsharpSelected",
"title": "C# (selected)"
},
{
"command": "codeQLDatabases.displayGo",
"title": "Go"
},
{
"command": "codeQLDatabases.displayGoSelected",
"title": "Go (selected)"
},
{
"command": "codeQLDatabases.displayJava",
"title": "Java/Kotlin"
},
{
"command": "codeQLDatabases.displayJavaSelected",
"title": "Java/Kotlin (selected)"
},
{
"command": "codeQLDatabases.displayJavascript",
"title": "JavaScript/TypeScript"
},
{
"command": "codeQLDatabases.displayJavascriptSelected",
"title": "JavaScript/TypeScript (selected)"
},
{
"command": "codeQLDatabases.displayPython",
"title": "Python"
},
{
"command": "codeQLDatabases.displayPythonSelected",
"title": "Python (selected)"
},
{
"command": "codeQLDatabases.displayRuby",
"title": "Ruby"
},
{
"command": "codeQLDatabases.displayRubySelected",
"title": "Ruby (selected)"
},
{
"command": "codeQLDatabases.displaySwift",
"title": "Swift"
},
{
"command": "codeQLDatabases.displaySwiftSelected",
"title": "Swift (selected)"
},
{
"command": "codeQL.chooseDatabaseFolder",
"title": "CodeQL: Choose Database from Folder"
@@ -778,19 +850,11 @@
},
{
"command": "codeQLDatabases.sortByName",
"title": "Sort by Name",
"icon": {
"light": "media/light/sort-alpha.svg",
"dark": "media/dark/sort-alpha.svg"
}
"title": "Sort by Name"
},
{
"command": "codeQLDatabases.sortByDateAdded",
"title": "Sort by Date Added",
"icon": {
"light": "media/light/sort-date.svg",
"dark": "media/dark/sort-date.svg"
}
"title": "Sort by Date Added"
},
{
"command": "codeQL.checkForUpdatesToCLI",
@@ -988,16 +1052,6 @@
}
],
"view/title": [
{
"command": "codeQLDatabases.sortByName",
"when": "view == codeQLDatabases",
"group": "navigation"
},
{
"command": "codeQLDatabases.sortByDateAdded",
"when": "view == codeQLDatabases",
"group": "navigation"
},
{
"command": "codeQLDatabases.chooseDatabaseFolder",
"when": "view == codeQLDatabases",
@@ -1018,6 +1072,21 @@
"when": "view == codeQLDatabases",
"group": "navigation"
},
{
"command": "codeQLDatabases.sortByName",
"when": "view == codeQLDatabases",
"group": "1_databases@0"
},
{
"command": "codeQLDatabases.sortByDateAdded",
"when": "view == codeQLDatabases",
"group": "1_databases@1"
},
{
"submenu": "codeQLDatabases.languages",
"when": "view == codeQLDatabases && config.codeQL.canary && config.codeQL.showLanguageFilter",
"group": "2_databases@0"
},
{
"command": "codeQLQueries.createQuery",
"when": "view == codeQLQueries",
@@ -1538,6 +1607,78 @@
"command": "codeQLDatabases.upgradeDatabase",
"when": "false"
},
{
"command": "codeQLDatabases.displayAllLanguages",
"when": "false"
},
{
"command": "codeQLDatabases.displayAllLanguagesSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayCpp",
"when": "false"
},
{
"command": "codeQLDatabases.displayCppSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayCsharp",
"when": "false"
},
{
"command": "codeQLDatabases.displayCsharpSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayGo",
"when": "false"
},
{
"command": "codeQLDatabases.displayGoSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayJava",
"when": "false"
},
{
"command": "codeQLDatabases.displayJavaSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayJavascript",
"when": "false"
},
{
"command": "codeQLDatabases.displayJavascriptSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayPython",
"when": "false"
},
{
"command": "codeQLDatabases.displayPythonSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayRuby",
"when": "false"
},
{
"command": "codeQLDatabases.displayRubySelected",
"when": "false"
},
{
"command": "codeQLDatabases.displaySwift",
"when": "false"
},
{
"command": "codeQLDatabases.displaySwiftSelected",
"when": "false"
},
{
"command": "codeQLQueryHistory.openQueryContextMenu",
"when": "false"
@@ -1732,8 +1873,88 @@
"command": "codeQL.gotoQLContextEditor",
"when": "editorLangId == ql-summary && config.codeQL.canary"
}
],
"codeQLDatabases.languages": [
{
"command": "codeQLDatabases.displayAllLanguages",
"when": "codeQLDatabases.languageFilter"
},
{
"command": "codeQLDatabases.displayAllLanguagesSelected",
"when": "!codeQLDatabases.languageFilter"
},
{
"command": "codeQLDatabases.displayCpp",
"when": "codeQLDatabases.languageFilter != cpp"
},
{
"command": "codeQLDatabases.displayCppSelected",
"when": "codeQLDatabases.languageFilter == cpp"
},
{
"command": "codeQLDatabases.displayCsharp",
"when": "codeQLDatabases.languageFilter != csharp"
},
{
"command": "codeQLDatabases.displayCsharpSelected",
"when": "codeQLDatabases.languageFilter == csharp"
},
{
"command": "codeQLDatabases.displayGo",
"when": "codeQLDatabases.languageFilter != go"
},
{
"command": "codeQLDatabases.displayGoSelected",
"when": "codeQLDatabases.languageFilter == go"
},
{
"command": "codeQLDatabases.displayJava",
"when": "codeQLDatabases.languageFilter != java"
},
{
"command": "codeQLDatabases.displayJavaSelected",
"when": "codeQLDatabases.languageFilter == java"
},
{
"command": "codeQLDatabases.displayJavascript",
"when": "codeQLDatabases.languageFilter != javascript"
},
{
"command": "codeQLDatabases.displayJavascriptSelected",
"when": "codeQLDatabases.languageFilter == javascript"
},
{
"command": "codeQLDatabases.displayPython",
"when": "codeQLDatabases.languageFilter != python"
},
{
"command": "codeQLDatabases.displayPythonSelected",
"when": "codeQLDatabases.languageFilter == python"
},
{
"command": "codeQLDatabases.displayRuby",
"when": "codeQLDatabases.languageFilter != ruby"
},
{
"command": "codeQLDatabases.displayRubySelected",
"when": "codeQLDatabases.languageFilter == ruby"
},
{
"command": "codeQLDatabases.displaySwift",
"when": "codeQLDatabases.languageFilter != swift"
},
{
"command": "codeQLDatabases.displaySwiftSelected",
"when": "codeQLDatabases.languageFilter == swift"
}
]
},
"submenus": [
{
"id": "codeQLDatabases.languages",
"label": "Languages"
}
],
"viewsContainers": {
"activitybar": [
{
@@ -1777,6 +1998,12 @@
"id": "codeQLEvalLogViewer",
"name": "Evaluator Log Viewer",
"when": "config.codeQL.canary"
},
{
"id": "codeQLMethodModeling",
"type": "webview",
"name": "CodeQL Method Modeling",
"when": "config.codeQL.canary && config.codeQL.model.methodModelingView && codeql.modelEditorOpen && !codeql.modelEditorActive"
}
],
"codeql-methods-usage": [
@@ -1785,14 +2012,6 @@
"name": "CodeQL Methods Usage",
"when": "config.codeQL.canary && codeql.modelEditorOpen"
}
],
"explorer": [
{
"type": "webview",
"id": "codeQLMethodModeling",
"name": "CodeQL Method Modeling",
"when": "config.codeQL.canary && config.codeQL.model.methodModelingView && codeql.modelEditorOpen"
}
]
},
"viewsWelcome": [
@@ -1849,13 +2068,15 @@
"storybook": "storybook dev -p 6006",
"build-storybook": "storybook build",
"lint:scenarios": "ts-node scripts/lint-scenarios.ts",
"generate": "npm-run-all -p generate:*",
"generate:schemas": "ts-node scripts/generate-schemas.ts",
"check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
"postinstall": "patch-package",
"prepare": "cd ../.. && husky install"
},
"dependencies": {
"@octokit/plugin-retry": "^4.1.6",
"@octokit/rest": "^19.0.4",
"@octokit/plugin-retry": "^6.0.1",
"@octokit/rest": "^20.0.2",
"@vscode/codicons": "^0.0.31",
"@vscode/debugadapter": "^1.59.0",
"@vscode/debugprotocol": "^1.59.0",
@@ -1869,10 +2090,10 @@
"fs-extra": "^11.1.1",
"immutable": "^4.0.0",
"js-yaml": "^4.1.0",
"msw": "^1.2.0",
"nanoid": "^3.2.0",
"msw": "^0.0.0-fetch.rc-20",
"nanoid": "^5.0.1",
"node-fetch": "^2.6.7",
"p-queue": "^6.0.0",
"p-queue": "^7.4.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"semver": "^7.5.2",
@@ -1889,7 +2110,7 @@
"vscode-languageclient": "^8.0.2",
"vscode-test-adapter-api": "^1.7.0",
"vscode-test-adapter-util": "^0.7.0",
"zip-a-folder": "^2.0.0"
"zip-a-folder": "^3.1.3"
},
"devDependencies": {
"@babel/core": "^7.18.13",
@@ -1899,7 +2120,7 @@
"@babel/preset-typescript": "^7.21.4",
"@faker-js/faker": "^8.0.2",
"@github/markdownlint-github": "^0.3.0",
"@octokit/plugin-throttling": "^5.0.1",
"@octokit/plugin-throttling": "^8.0.0",
"@storybook/addon-actions": "^7.1.0",
"@storybook/addon-essentials": "^7.1.0",
"@storybook/addon-interactions": "^7.1.0",
@@ -1923,9 +2144,9 @@
"@types/gulp": "^4.0.9",
"@types/gulp-replace": "^1.1.0",
"@types/jest": "^29.0.2",
"@types/js-yaml": "^3.12.5",
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "^16.11.25",
"@types/node": "18.15.0",
"@types/node-fetch": "^2.5.2",
"@types/react": "^18.0.28",
"@types/react-dom": "^18.0.11",
@@ -1937,7 +2158,7 @@
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "^0.10.1",
"@types/vscode": "^1.67.0",
"@types/vscode": "^1.82.0",
"@types/webpack": "^5.28.0",
"@types/webpack-env": "^1.18.0",
"@typescript-eslint/eslint-plugin": "^6.2.1",

6
extensions/ql-vscode/scripts/add-fields-to-scenarios.ts
View File

@@ -14,7 +14,8 @@
import { pathExists, readJson, writeJson } from "fs-extra";
import { resolve, relative } from "path";
import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
import { Octokit } from "@octokit/core";
import { type RestEndpointMethodTypes } from "@octokit/rest";
import { throttling } from "@octokit/plugin-throttling";
import { getFiles } from "./util/files";
@@ -22,6 +23,7 @@ import type { GitHubApiRequest } from "../src/common/mock-gh-api/gh-api-request"
import { isGetVariantAnalysisRequest } from "../src/common/mock-gh-api/gh-api-request";
import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
import { AppOctokit } from "../src/common/octokit";
const extensionDirectory = resolve(__dirname, "..");
const scenariosDirectory = resolve(
@@ -31,7 +33,7 @@ const scenariosDirectory = resolve(
// Make sure we don't run into rate limits by automatically waiting until we can
// make another request.
const MyOctokit = Octokit.plugin(throttling);
const MyOctokit = AppOctokit.plugin(throttling);
const auth = process.env.GITHUB_TOKEN;

72
extensions/ql-vscode/scripts/generate-schemas.ts Normal file
View File

@@ -0,0 +1,72 @@
import { createGenerator } from "ts-json-schema-generator";
import { join, resolve } from "path";
import { outputFile } from "fs-extra";
import { format, resolveConfig } from "prettier";
const extensionDirectory = resolve(__dirname, "..");
const schemas = [
{
path: join(
extensionDirectory,
"src",
"model-editor",
"extension-pack-metadata.ts",
),
type: "ExtensionPackMetadata",
schemaPath: join(
extensionDirectory,
"src",
"model-editor",
"extension-pack-metadata.schema.json",
),
},
{
path: join(
extensionDirectory,
"src",
"model-editor",
"model-extension-file.ts",
),
type: "ModelExtensionFile",
schemaPath: join(
extensionDirectory,
"src",
"model-editor",
"model-extension-file.schema.json",
),
},
];
async function generateSchema(
schemaDefinition: (typeof schemas)[number],
): Promise<void> {
const schema = createGenerator({
path: schemaDefinition.path,
tsconfig: resolve(extensionDirectory, "tsconfig.json"),
type: schemaDefinition.type,
skipTypeCheck: true,
topRef: true,
additionalProperties: true,
}).createSchema(schemaDefinition.type);
const schemaJson = JSON.stringify(schema, null, 2);
const prettierOptions = await resolveConfig(schemaDefinition.schemaPath);
const formattedSchemaJson = await format(schemaJson, {
...prettierOptions,
filepath: schemaDefinition.schemaPath,
});
await outputFile(schemaDefinition.schemaPath, formattedSchemaJson);
}
async function generateSchemas() {
await Promise.all(schemas.map(generateSchema));
}
generateSchemas().catch((e: unknown) => {
console.error(e);
process.exit(2);
});

111
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -6,7 +6,6 @@ import { dirname, join, delimiter } from "path";
import * as sarif from "sarif";
import { SemVer } from "semver";
import { Readable } from "stream";
import { StringDecoder } from "string_decoder";
import tk from "tree-kill";
import { promisify } from "util";
import { CancellationToken, Disposable, Uri } from "vscode";
@@ -31,6 +30,7 @@ import { CompilationMessage } from "../query-server/legacy-messages";
import { sarifParser } from "../common/sarif-parser";
import { App } from "../common/app";
import { QueryLanguage } from "../common/query-language";
import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
/**
* The version of the SARIF format that we are using.
@@ -1649,120 +1649,13 @@ export async function runCodeQlCliCommand(
}
}
/**
* Buffer to hold state used when splitting a text stream into lines.
*/
class SplitBuffer {
private readonly decoder = new StringDecoder("utf8");
private readonly maxSeparatorLength: number;
private buffer = "";
private searchIndex = 0;
constructor(private readonly separators: readonly string[]) {
this.maxSeparatorLength = separators
.map((s) => s.length)
.reduce((a, b) => Math.max(a, b), 0);
}
/**
* Append new text data to the buffer.
* @param chunk The chunk of data to append.
*/
public addChunk(chunk: Buffer): void {
this.buffer += this.decoder.write(chunk);
}
/**
* Signal that the end of the input stream has been reached.
*/
public end(): void {
this.buffer += this.decoder.end();
this.buffer += this.separators[0]; // Append a separator to the end to ensure the last line is returned.
}
/**
* A version of startsWith that isn't overriden by a broken version of ms-python.
*
* The definition comes from
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
* which is CC0/public domain
*
* See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
*/
private static startsWith(
s: string,
searchString: string,
position: number,
): boolean {
const pos = position > 0 ? position | 0 : 0;
return s.substring(pos, pos + searchString.length) === searchString;
}
/**
* Extract the next full line from the buffer, if one is available.
* @returns The text of the next available full line (without the separator), or `undefined` if no
* line is available.
*/
public getNextLine(): string | undefined {
while (this.searchIndex <= this.buffer.length - this.maxSeparatorLength) {
for (const separator of this.separators) {
if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
const line = this.buffer.slice(0, this.searchIndex);
this.buffer = this.buffer.slice(this.searchIndex + separator.length);
this.searchIndex = 0;
return line;
}
}
this.searchIndex++;
}
return undefined;
}
}
/**
* Splits a text stream into lines based on a list of valid line separators.
* @param stream The text stream to split. This stream will be fully consumed.
* @param separators The list of strings that act as line separators.
* @returns A sequence of lines (not including separators).
*/
async function* splitStreamAtSeparators(
stream: Readable,
separators: string[],
): AsyncGenerator<string, void, unknown> {
const buffer = new SplitBuffer(separators);
for await (const chunk of stream) {
buffer.addChunk(chunk);
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
buffer.end();
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
/**
* Standard line endings for splitting human-readable text.
*/
const lineEndings = ["\r\n", "\r", "\n"];
/**
* Log a text stream to a `Logger` interface.
* @param stream The stream to log.
* @param logger The logger that will consume the stream output.
*/
async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
for await (const line of splitStreamAtSeparators(stream, lineEndings)) {
for await (const line of splitStreamAtSeparators(stream, LINE_ENDINGS)) {
// Await the result of log here in order to ensure the logs are written in the correct order.
await logger.log(line);
}

18
extensions/ql-vscode/src/common/commands.ts
View File

@@ -219,6 +219,24 @@ export type LocalDatabasesCommands = {
"codeQLDatabases.chooseDatabaseGithub": () => Promise<void>;
"codeQLDatabases.sortByName": () => Promise<void>;
"codeQLDatabases.sortByDateAdded": () => Promise<void>;
"codeQLDatabases.displayAllLanguages": () => Promise<void>;
"codeQLDatabases.displayCpp": () => Promise<void>;
"codeQLDatabases.displayCsharp": () => Promise<void>;
"codeQLDatabases.displayGo": () => Promise<void>;
"codeQLDatabases.displayJava": () => Promise<void>;
"codeQLDatabases.displayJavascript": () => Promise<void>;
"codeQLDatabases.displayPython": () => Promise<void>;
"codeQLDatabases.displayRuby": () => Promise<void>;
"codeQLDatabases.displaySwift": () => Promise<void>;
"codeQLDatabases.displayAllLanguagesSelected": () => Promise<void>;
"codeQLDatabases.displayCppSelected": () => Promise<void>;
"codeQLDatabases.displayCsharpSelected": () => Promise<void>;
"codeQLDatabases.displayGoSelected": () => Promise<void>;
"codeQLDatabases.displayJavaSelected": () => Promise<void>;
"codeQLDatabases.displayJavascriptSelected": () => Promise<void>;
"codeQLDatabases.displayPythonSelected": () => Promise<void>;
"codeQLDatabases.displayRubySelected": () => Promise<void>;
"codeQLDatabases.displaySwiftSelected": () => Promise<void>;
// Database panel context menu
"codeQLDatabases.setCurrentDatabase": (

8
extensions/ql-vscode/src/common/disposable-object.ts
View File

@@ -9,10 +9,16 @@ export type DisposeHandler = (disposable: Disposable) => void;
/**
* Base class to make it easier to implement a `Disposable` that owns other disposable object.
*/
export abstract class DisposableObject implements Disposable {
export class DisposableObject implements Disposable {
private disposables: Disposable[] = [];
private tracked?: Set<Disposable> = undefined;
constructor(...dispoables: Disposable[]) {
for (const d of dispoables) {
this.push(d);
}
}
/**
* Adds `obj` to a list of objects to dispose when `this` is disposed. Objects added by `push` are
* disposed in reverse order of being added.

62
extensions/ql-vscode/src/common/interface-types.ts
View File

@@ -21,6 +21,7 @@ import { Method, Usage } from "../model-editor/method";
import { ModeledMethod } from "../model-editor/modeled-method";
import { ModelEditorViewState } from "../model-editor/shared/view-state";
import { Mode } from "../model-editor/shared/mode";
import { QueryLanguage } from "./query-language";
/**
* This module contains types and code that are shared between
@@ -51,6 +52,7 @@ export const RAW_RESULTS_LIMIT = 10000;
export interface DatabaseInfo {
name: string;
databaseUri: string;
language?: QueryLanguage;
}
/** Arbitrary query metadata */
@@ -500,14 +502,14 @@ interface SetMethodsMessage {
methods: Method[];
}
interface LoadModeledMethodsMessage {
t: "loadModeledMethods";
modeledMethods: Record<string, ModeledMethod>;
interface SetModeledMethodsMessage {
t: "setModeledMethods";
methods: Record<string, ModeledMethod>;
}
interface AddModeledMethodsMessage {
t: "addModeledMethods";
modeledMethods: Record<string, ModeledMethod>;
interface SetModifiedMethodsMessage {
t: "setModifiedMethods";
methodSignatures: string[];
}
interface SetInProgressMethodsMessage {
@@ -570,12 +572,23 @@ interface HideModeledMethodsMessage {
hideModeledMethods: boolean;
}
interface SetModeledMethodMessage {
t: "setModeledMethod";
method: ModeledMethod;
}
interface RevealMethodMessage {
t: "revealMethod";
method: Method;
}
export type ToModelEditorMessage =
| SetExtensionPackStateMessage
| SetMethodsMessage
| LoadModeledMethodsMessage
| AddModeledMethodsMessage
| SetInProgressMethodsMessage;
| SetModeledMethodsMessage
| SetModifiedMethodsMessage
| SetInProgressMethodsMessage
| RevealMethodMessage;
export type FromModelEditorMessage =
| ViewLoadedMsg
@@ -589,15 +602,38 @@ export type FromModelEditorMessage =
| GenerateMethodsFromLlmMessage
| StopGeneratingMethodsFromLlmMessage
| ModelDependencyMessage
| HideModeledMethodsMessage;
| HideModeledMethodsMessage
| SetModeledMethodMessage;
interface RevealInEditorMessage {
t: "revealInModelEditor";
method: Method;
}
export type FromMethodModelingMessage =
| TelemetryMessage
| UnhandledErrorMessage;
| CommonFromViewMessages
| SetModeledMethodMessage
| RevealInEditorMessage;
interface SetMethodMessage {
t: "setMethod";
method: Method;
}
export type ToMethodModelingMessage = SetMethodMessage;
interface SetMethodModifiedMessage {
t: "setMethodModified";
isModified: boolean;
}
interface SetSelectedMethodMessage {
t: "setSelectedMethod";
method: Method;
modeledMethod?: ModeledMethod;
isModified: boolean;
}
export type ToMethodModelingMessage =
| SetMethodMessage
| SetModeledMethodMessage
| SetMethodModifiedMessage
| SetSelectedMethodMessage;

34
extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts
View File

@@ -17,7 +17,7 @@ export enum RequestKind {
AutoModel = "autoModel",
}
interface BasicErorResponse {
export interface BasicErrorResponse {
message: string;
}
@@ -27,7 +27,7 @@ interface GetRepoRequest {
};
response: {
status: number;
body: Repository | BasicErorResponse | undefined;
body: Repository | BasicErrorResponse | undefined;
};
}
@@ -37,7 +37,7 @@ interface SubmitVariantAnalysisRequest {
};
response: {
status: number;
body?: VariantAnalysis | BasicErorResponse;
body?: VariantAnalysis | BasicErrorResponse;
};
}
@@ -47,7 +47,7 @@ interface GetVariantAnalysisRequest {
};
response: {
status: number;
body?: VariantAnalysis | BasicErorResponse;
body?: VariantAnalysis | BasicErrorResponse;
};
}
@@ -58,7 +58,7 @@ interface GetVariantAnalysisRepoRequest {
};
response: {
status: number;
body?: VariantAnalysisRepoTask | BasicErorResponse;
body?: VariantAnalysisRepoTask | BasicErrorResponse;
};
}
@@ -74,6 +74,13 @@ export interface GetVariantAnalysisRepoResultRequest {
};
}
export interface CodeSearchResponse {
total_count: number;
items: Array<{
repository: Repository;
}>;
}
interface CodeSearchRequest {
request: {
kind: RequestKind.CodeSearch;
@@ -81,16 +88,14 @@ interface CodeSearchRequest {
};
response: {
status: number;
body?: {
total_count?: number;
items?: Array<{
repository: Repository;
}>;
};
message?: string;
body?: CodeSearchResponse | BasicErrorResponse;
};
}
export interface AutoModelResponse {
models: string;
}
interface AutoModelRequest {
request: {
kind: RequestKind.AutoModel;
@@ -100,10 +105,7 @@ interface AutoModelRequest {
};
response: {
status: number;
body?: {
models: string;
};
message?: string;
body?: AutoModelResponse | BasicErrorResponse;
};
}

106
extensions/ql-vscode/src/common/mock-gh-api/recorder.ts
View File

@@ -1,30 +1,33 @@
import { ensureDir, writeFile } from "fs-extra";
import { join } from "path";
import { MockedRequest } from "msw";
import { SetupServer } from "msw/node";
import { IsomorphicResponse } from "@mswjs/interceptors";
import { Headers } from "headers-polyfill";
import fetch from "node-fetch";
import { SetupServer } from "msw/node";
import { DisposableObject } from "../disposable-object";
import { gzipDecode } from "../zlib";
import {
AutoModelResponse,
BasicErrorResponse,
CodeSearchResponse,
GetVariantAnalysisRepoResultRequest,
GitHubApiRequest,
RequestKind,
} from "./gh-api-request";
import {
VariantAnalysis,
VariantAnalysisRepoTask,
} from "../../variant-analysis/gh-api/variant-analysis";
import { Repository } from "../../variant-analysis/gh-api/repository";
export class Recorder extends DisposableObject {
private readonly allRequests = new Map<string, MockedRequest>();
private currentRecordedScenario: GitHubApiRequest[] = [];
private _isRecording = false;
constructor(private readonly server: SetupServer) {
super();
this.onRequestStart = this.onRequestStart.bind(this);
this.onResponseBypass = this.onResponseBypass.bind(this);
}
@@ -45,7 +48,6 @@ export class Recorder extends DisposableObject {
this.clear();
this.server.events.on("request:start", this.onRequestStart);
this.server.events.on("response:bypass", this.onResponseBypass);
}
@@ -56,13 +58,11 @@ export class Recorder extends DisposableObject {
this._isRecording = false;
this.server.events.removeListener("request:start", this.onRequestStart);
this.server.events.removeListener("response:bypass", this.onResponseBypass);
}
public clear() {
this.currentRecordedScenario = [];
this.allRequests.clear();
}
public async save(scenariosPath: string, name: string): Promise<string> {
@@ -91,7 +91,7 @@ export class Recorder extends DisposableObject {
let bodyFileLink = undefined;
if (writtenRequest.response.body) {
await writeFile(bodyFilePath, writtenRequest.response.body || "");
await writeFile(bodyFilePath, writtenRequest.response.body);
bodyFileLink = `file:${bodyFileName}`;
}
@@ -112,33 +112,18 @@ export class Recorder extends DisposableObject {
return scenarioDirectory;
}
private onRequestStart(request: MockedRequest): void {
private async onResponseBypass(
response: Response,
request: Request,
_requestId: string,
): Promise<void> {
if (request.headers.has("x-vscode-codeql-msw-bypass")) {
return;
}
this.allRequests.set(request.id, request);
}
private async onResponseBypass(
response: IsomorphicResponse,
requestId: string,
): Promise<void> {
const request = this.allRequests.get(requestId);
this.allRequests.delete(requestId);
if (!request) {
return;
}
if (response.body === undefined) {
return;
}
const gitHubApiRequest = await createGitHubApiRequest(
request.url.toString(),
response.status,
response.body,
response.headers,
request.url,
response,
);
if (!gitHubApiRequest) {
return;
@@ -150,14 +135,14 @@ export class Recorder extends DisposableObject {
async function createGitHubApiRequest(
url: string,
status: number,
body: string,
headers: Headers,
response: Response,
): Promise<GitHubApiRequest | undefined> {
if (!url) {
return undefined;
}
const status = response.status;
if (url.match(/\/repos\/[a-zA-Z0-9-_.]+\/[a-zA-Z0-9-_.]+$/)) {
return {
request: {
@@ -165,7 +150,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
Repository | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -179,7 +166,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
VariantAnalysis | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -195,7 +184,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
VariantAnalysis | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -211,7 +202,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
VariantAnalysisRepoTask | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -238,9 +231,10 @@ async function createGitHubApiRequest(
repositoryId: parseInt(repoDownloadMatch.groups.repositoryId, 10),
},
response: {
status,
status: response.status,
body: responseBuffer,
contentType: headers.get("content-type") ?? "application/octet-stream",
contentType:
response.headers.get("content-type") ?? "application/octet-stream",
},
};
}
@@ -254,7 +248,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
CodeSearchResponse | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -269,7 +265,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
BasicErrorResponse | AutoModelResponse | undefined
>(response),
},
};
}
@@ -277,6 +275,26 @@ async function createGitHubApiRequest(
return undefined;
}
async function responseBody(response: Response): Promise<Uint8Array> {
const body = await response.arrayBuffer();
const view = new Uint8Array(body);
if (view[0] === 0x1f && view[1] === 0x8b) {
// Response body is gzipped, so we need to un-gzip it.
return await gzipDecode(view);
} else {
return view;
}
}
async function jsonResponseBody<T>(response: Response): Promise<T> {
const body = await responseBody(response);
const text = new TextDecoder("utf-8").decode(body);
return JSON.parse(text);
}
function shouldWriteBodyToFile(
request: GitHubApiRequest,
): request is GetVariantAnalysisRepoResultRequest {

93
extensions/ql-vscode/src/common/mock-gh-api/request-handlers.ts
View File

@@ -1,6 +1,6 @@
import { join } from "path";
import { readdir, readJson, readFile } from "fs-extra";
import { DefaultBodyType, MockedRequest, rest, RestHandler } from "msw";
import { RequestHandler, rest } from "msw";
import {
GitHubApiRequest,
isAutoModelRequest,
@@ -14,7 +14,19 @@ import {
const baseUrl = "https://api.github.com";
type RequestHandler = RestHandler<MockedRequest<DefaultBodyType>>;
const jsonResponse = <T>(
body: T,
init?: ResponseInit,
contentType = "application/json",
): Response => {
return new Response(JSON.stringify(body), {
...init,
headers: {
"Content-Type": contentType,
...init?.headers,
},
});
};
export async function createRequestHandlers(
scenarioDirPath: string,
@@ -82,11 +94,10 @@ function createGetRepoRequestHandler(
const getRepoRequest = getRepoRequests[0];
return rest.get(`${baseUrl}/repos/:owner/:name`, (_req, res, ctx) => {
return res(
ctx.status(getRepoRequest.response.status),
ctx.json(getRepoRequest.response.body),
);
return rest.get(`${baseUrl}/repos/:owner/:name`, () => {
return jsonResponse(getRepoRequest.response.body, {
status: getRepoRequest.response.status,
});
});
}
@@ -105,11 +116,10 @@ function createSubmitVariantAnalysisRequestHandler(
return rest.post(
`${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses`,
(_req, res, ctx) => {
return res(
ctx.status(getRepoRequest.response.status),
ctx.json(getRepoRequest.response.body),
);
() => {
return jsonResponse(getRepoRequest.response.body, {
status: getRepoRequest.response.status,
});
},
);
}
@@ -127,7 +137,7 @@ function createGetVariantAnalysisRequestHandler(
// request, so keep an index of the request and return the appropriate response.
return rest.get(
`${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId`,
(_req, res, ctx) => {
() => {
const request = getVariantAnalysisRequests[requestIndex];
if (requestIndex < getVariantAnalysisRequests.length - 1) {
@@ -135,10 +145,9 @@ function createGetVariantAnalysisRequestHandler(
requestIndex++;
}
return res(
ctx.status(request.response.status),
ctx.json(request.response.body),
);
return jsonResponse(request.response.body, {
status: request.response.status,
});
},
);
}
@@ -152,18 +161,17 @@ function createGetVariantAnalysisRepoRequestHandler(
return rest.get(
`${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId/repositories/:repoId`,
(req, res, ctx) => {
({ request, params }) => {
const scenarioRequest = getVariantAnalysisRepoRequests.find(
(r) => r.request.repositoryId.toString() === req.params.repoId,
(r) => r.request.repositoryId.toString() === params.repoId,
);
if (!scenarioRequest) {
throw Error(`No scenario request found for ${req.url}`);
throw Error(`No scenario request found for ${request.url}`);
}
return res(
ctx.status(scenarioRequest.response.status),
ctx.json(scenarioRequest.response.body),
);
return jsonResponse(scenarioRequest.response.body, {
status: scenarioRequest.response.status,
});
},
);
}
@@ -177,22 +185,23 @@ function createGetVariantAnalysisRepoResultRequestHandler(
return rest.get(
"https://objects-origin.githubusercontent.com/codeql-query-console/codeql-variant-analysis-repo-tasks/:variantAnalysisId/:repoId/*",
(req, res, ctx) => {
({ request, params }) => {
const scenarioRequest = getVariantAnalysisRepoResultRequests.find(
(r) => r.request.repositoryId.toString() === req.params.repoId,
(r) => r.request.repositoryId.toString() === params.repoId,
);
if (!scenarioRequest) {
throw Error(`No scenario request found for ${req.url}`);
throw Error(`No scenario request found for ${request.url}`);
}
if (scenarioRequest.response.body) {
return res(
ctx.status(scenarioRequest.response.status),
ctx.set("Content-Type", scenarioRequest.response.contentType),
ctx.body(scenarioRequest.response.body),
);
return new Response(scenarioRequest.response.body, {
status: scenarioRequest.response.status,
headers: {
"Content-Type": scenarioRequest.response.contentType,
},
});
} else {
return res(ctx.status(scenarioRequest.response.status));
return new Response(null, { status: scenarioRequest.response.status });
}
},
);
@@ -207,7 +216,7 @@ function createCodeSearchRequestHandler(
// During a code search, there are multiple request to get pages of results. We
// need to return different responses for each request, so keep an index of the
// request and return the appropriate response.
return rest.get(`${baseUrl}/search/code?q=*`, (_req, res, ctx) => {
return rest.get(`${baseUrl}/search/code`, () => {
const request = codeSearchRequests[requestIndex];
if (requestIndex < codeSearchRequests.length - 1) {
@@ -215,10 +224,9 @@ function createCodeSearchRequestHandler(
requestIndex++;
}
return res(
ctx.status(request.response.status),
ctx.json(request.response.body),
);
return jsonResponse(request.response.body, {
status: request.response.status,
});
});
}
@@ -233,7 +241,7 @@ function createAutoModelRequestHandler(
// so keep an index of the request and return the appropriate response.
return rest.post(
`${baseUrl}/repos/github/codeql/code-scanning/codeql/auto-model`,
(_req, res, ctx) => {
() => {
const request = autoModelRequests[requestIndex];
if (requestIndex < autoModelRequests.length - 1) {
@@ -241,10 +249,9 @@ function createAutoModelRequestHandler(
requestIndex++;
}
return res(
ctx.status(request.response.status),
ctx.json(request.response.body),
);
return jsonResponse(request.response.body, {
status: request.response.status,
});
},
);
}

10
extensions/ql-vscode/src/common/octokit.ts Normal file
View File

@@ -0,0 +1,10 @@
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
import fetch from "node-fetch";
export const AppOctokit = Octokit.Octokit.defaults({
request: {
fetch,
},
retry,
});

6
extensions/ql-vscode/src/common/query-language.ts
View File

@@ -62,3 +62,9 @@ export const dbSchemeToLanguage: Record<string, QueryLanguage> = {
export function isQueryLanguage(language: string): language is QueryLanguage {
return Object.values(QueryLanguage).includes(language as QueryLanguage);
}
export function tryGetQueryLanguage(
language: string,
): QueryLanguage | undefined {
return isQueryLanguage(language) ? language : undefined;
}

125
extensions/ql-vscode/src/common/split-stream.ts Normal file
View File

@@ -0,0 +1,125 @@
import { Readable } from "stream";
import { StringDecoder } from "string_decoder";
/**
* Buffer to hold state used when splitting a text stream into lines.
*/
export class SplitBuffer {
private readonly decoder = new StringDecoder("utf8");
private readonly maxSeparatorLength: number;
private buffer = "";
private searchIndex = 0;
private ended = false;
constructor(private readonly separators: readonly string[]) {
this.maxSeparatorLength = separators
.map((s) => s.length)
.reduce((a, b) => Math.max(a, b), 0);
}
/**
* Append new text data to the buffer.
* @param chunk The chunk of data to append.
*/
public addChunk(chunk: Buffer): void {
this.buffer += this.decoder.write(chunk);
}
/**
* Signal that the end of the input stream has been reached.
*/
public end(): void {
this.buffer += this.decoder.end();
this.ended = true;
}
/**
* A version of startsWith that isn't overriden by a broken version of ms-python.
*
* The definition comes from
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
* which is CC0/public domain
*
* See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
*/
private static startsWith(
s: string,
searchString: string,
position: number,
): boolean {
const pos = position > 0 ? position | 0 : 0;
return s.substring(pos, pos + searchString.length) === searchString;
}
/**
* Extract the next full line from the buffer, if one is available.
* @returns The text of the next available full line (without the separator), or `undefined` if no
* line is available.
*/
public getNextLine(): string | undefined {
// If we haven't received all of the input yet, don't search too close to the end of the buffer,
// or we could match a separator that's split across two chunks. For example, we could see "\r"
// at the end of the buffer and match that, even though we were about to receive a "\n" right
// after it.
const maxSearchIndex = this.ended
? this.buffer.length - 1
: this.buffer.length - this.maxSeparatorLength;
while (this.searchIndex <= maxSearchIndex) {
for (const separator of this.separators) {
if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
const line = this.buffer.slice(0, this.searchIndex);
this.buffer = this.buffer.slice(this.searchIndex + separator.length);
this.searchIndex = 0;
return line;
}
}
this.searchIndex++;
}
if (this.ended && this.buffer.length > 0) {
// If we still have some text left in the buffer, return it as the last line.
const line = this.buffer;
this.buffer = "";
this.searchIndex = 0;
return line;
} else {
return undefined;
}
}
}
/**
* Splits a text stream into lines based on a list of valid line separators.
* @param stream The text stream to split. This stream will be fully consumed.
* @param separators The list of strings that act as line separators.
* @returns A sequence of lines (not including separators).
*/
export async function* splitStreamAtSeparators(
stream: Readable,
separators: string[],
): AsyncGenerator<string, void, unknown> {
const buffer = new SplitBuffer(separators);
for await (const chunk of stream) {
buffer.addChunk(chunk);
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
buffer.end();
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
/**
* Standard line endings for splitting human-readable text.
*/
export const LINE_ENDINGS = ["\r\n", "\r", "\n"];

85
extensions/ql-vscode/src/common/vscode/abstract-webview-view-provider.ts Normal file
View File

@@ -0,0 +1,85 @@
import * as vscode from "vscode";
import { Uri, WebviewViewProvider } from "vscode";
import { WebviewKind, WebviewMessage, getHtmlForWebview } from "./webview-html";
import { Disposable } from "../disposable-object";
import { App } from "../app";
export abstract class AbstractWebviewViewProvider<
ToMessage extends WebviewMessage,
FromMessage extends WebviewMessage,
> implements WebviewViewProvider
{
protected webviewView: vscode.WebviewView | undefined = undefined;
private disposables: Disposable[] = [];
constructor(
private readonly app: App,
private readonly webviewKind: WebviewKind,
) {}
/**
* This is called when a view first becomes visible. This may happen when the view is
* first loaded or when the user hides and then shows a view again.
*/
public resolveWebviewView(
webviewView: vscode.WebviewView,
_context: vscode.WebviewViewResolveContext,
_token: vscode.CancellationToken,
) {
webviewView.webview.options = {
enableScripts: true,
localResourceRoots: [Uri.file(this.app.extensionPath)],
};
const html = getHtmlForWebview(
this.app,
webviewView.webview,
this.webviewKind,
{
allowInlineStyles: true,
allowWasmEval: false,
},
);
webviewView.webview.html = html;
this.webviewView = webviewView;
webviewView.webview.onDidReceiveMessage(async (msg) => this.onMessage(msg));
webviewView.onDidDispose(() => this.dispose());
}
protected get isShowingView() {
return this.webviewView?.visible ?? false;
}
protected async postMessage(msg: ToMessage): Promise<void> {
await this.webviewView?.webview.postMessage(msg);
}
protected dispose() {
while (this.disposables.length > 0) {
const disposable = this.disposables.pop()!;
disposable.dispose();
}
this.webviewView = undefined;
}
protected push<T extends Disposable>(obj: T): T {
if (obj !== undefined) {
this.disposables.push(obj);
}
return obj;
}
protected abstract onMessage(msg: FromMessage): Promise<void>;
/**
* This is called when a view first becomes visible. This may happen when the view is
* first loaded or when the user hides and then shows a view again.
*/
protected onWebViewLoaded(): void {
// Do nothing by default.
}
}

34
extensions/ql-vscode/src/common/vscode/abstract-webview.ts
View File

@@ -9,7 +9,7 @@ import {
import { join } from "path";
import { App } from "../app";
import { DisposableObject, DisposeHandler } from "../disposable-object";
import { Disposable } from "../disposable-object";
import { tmpDir } from "../../tmp-dir";
import { getHtmlForWebview, WebviewMessage, WebviewKind } from "./webview-html";
@@ -27,16 +27,16 @@ export type WebviewPanelConfig = {
export abstract class AbstractWebview<
ToMessage extends WebviewMessage,
FromMessage extends WebviewMessage,
> extends DisposableObject {
> {
protected panel: WebviewPanel | undefined;
protected panelLoaded = false;
protected panelLoadedCallBacks: Array<() => void> = [];
private panelResolves?: Array<(panel: WebviewPanel) => void>;
constructor(protected readonly app: App) {
super();
}
private disposables: Disposable[] = [];
constructor(protected readonly app: App) {}
public async restoreView(panel: WebviewPanel): Promise<void> {
this.panel = panel;
@@ -101,6 +101,7 @@ export abstract class AbstractWebview<
this.panel = undefined;
this.panelLoaded = false;
this.onPanelDispose();
this.disposeAll();
}, null),
);
@@ -150,8 +151,27 @@ export abstract class AbstractWebview<
return panel.webview.postMessage(msg);
}
public dispose(disposeHandler?: DisposeHandler) {
public dispose() {
this.panel?.dispose();
super.dispose(disposeHandler);
this.disposeAll();
}
private disposeAll() {
while (this.disposables.length > 0) {
const disposable = this.disposables.pop()!;
disposable.dispose();
}
}
/**
* Adds `obj` to a list of objects to dispose when the panel is disposed. Objects added by `push` are
* disposed in reverse order of being added.
* @param obj The object to take ownership of.
*/
protected push<T extends Disposable>(obj: T): T {
if (obj !== undefined) {
this.disposables.push(obj);
}
return obj;
}
}

5
extensions/ql-vscode/src/common/vscode/authentication.ts
View File

@@ -1,7 +1,7 @@
import * as vscode from "vscode";
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
import { Credentials } from "../authentication";
import { AppOctokit } from "../octokit";
export const GITHUB_AUTH_PROVIDER_ID = "github";
@@ -32,9 +32,8 @@ export class VSCodeCredentials implements Credentials {
const accessToken = await this.getAccessToken();
return new Octokit.Octokit({
return new AppOctokit({
auth: accessToken,
retry,
});
}

24
extensions/ql-vscode/src/common/vscode/multi-cancellation-token.ts Normal file
View File

@@ -0,0 +1,24 @@
import { CancellationToken, Disposable } from "vscode";
import { DisposableObject } from "../disposable-object";
/**
* A cancellation token that cancels when any of its constituent
* cancellation tokens are cancelled.
*/
export class MultiCancellationToken implements CancellationToken {
private readonly tokens: CancellationToken[];
constructor(...tokens: CancellationToken[]) {
this.tokens = tokens;
}
get isCancellationRequested(): boolean {
return this.tokens.some((t) => t.isCancellationRequested);
}
onCancellationRequested<T>(listener: (e: T) => any): Disposable {
return new DisposableObject(
...this.tokens.map((t) => t.onCancellationRequested(listener)),
);
}
}

5
extensions/ql-vscode/src/config.ts
View File

@@ -703,6 +703,7 @@ const MODEL_SETTING = new Setting("model", ROOT_SETTING);
const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING);
const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING);
const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
const SHOW_MULTIPLE_MODELS = new Setting("showMultipleModels", MODEL_SETTING);
export function showFlowGeneration(): boolean {
return !!FLOW_GENERATION.getValue<boolean>();
@@ -717,3 +718,7 @@ export function getExtensionsDirectory(languageId: string): string | undefined {
languageId,
});
}
export function showMultipleModels(): boolean {
return !!SHOW_MULTIPLE_MODELS.getValue<boolean>();
}

20
extensions/ql-vscode/src/databases/code-search-api.ts
View File

@@ -1,12 +1,9 @@
import { retry } from "@octokit/plugin-retry";
import { throttling } from "@octokit/plugin-throttling";
import { Octokit } from "@octokit/rest";
import { Progress, CancellationToken } from "vscode";
import { Credentials } from "../common/authentication";
import {
NotificationLogger,
showAndLogWarningMessage,
} from "../common/logging";
import { BaseLogger } from "../common/logging";
import { AppOctokit } from "../common/octokit";
export async function getCodeSearchRepositories(
query: string,
@@ -16,7 +13,7 @@ export async function getCodeSearchRepositories(
}>,
token: CancellationToken,
credentials: Credentials,
logger: NotificationLogger,
logger: BaseLogger,
): Promise<string[]> {
let nwos: string[] = [];
const octokit = await provideOctokitWithThrottling(credentials, logger);
@@ -47,26 +44,23 @@ export async function getCodeSearchRepositories(
async function provideOctokitWithThrottling(
credentials: Credentials,
logger: NotificationLogger,
logger: BaseLogger,
): Promise<Octokit> {
const MyOctokit = Octokit.plugin(throttling);
const MyOctokit = AppOctokit.plugin(throttling);
const auth = await credentials.getAccessToken();
const octokit = new MyOctokit({
auth,
retry,
throttle: {
onRateLimit: (retryAfter: number, options: any): boolean => {
void showAndLogWarningMessage(
logger,
void logger.log(
`Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`,
);
return true;
},
onSecondaryRateLimit: (_retryAfter: number, options: any): void => {
void showAndLogWarningMessage(
logger,
void logger.log(
`Secondary Rate Limit detected for request ${options.method} ${options.url}`,
);
},

4
extensions/ql-vscode/src/databases/database-fetcher.ts
View File

@@ -14,7 +14,6 @@ import {
} from "fs-extra";
import { basename, join } from "path";
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
import { DatabaseManager, DatabaseItem } from "./local-databases";
import { tmpDir } from "../tmp-dir";
@@ -32,6 +31,7 @@ import { Credentials } from "../common/authentication";
import { AppCommandManager } from "../common/commands";
import { allowHttp } from "../config";
import { showAndLogInformationMessage } from "../common/logging";
import { AppOctokit } from "../common/octokit";
/**
* Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -186,7 +186,7 @@ export async function downloadGitHubDatabase(
const octokit = credentials
? await credentials.getOctokit()
: new Octokit.Octokit({ retry });
: new AppOctokit();
const result = await convertGithubNwoToDatabaseUrl(
nwo,

87
extensions/ql-vscode/src/databases/local-databases-ui.ts
View File

@@ -51,6 +51,8 @@ import {
createMultiSelectionCommand,
createSingleSelectionCommand,
} from "../common/vscode/selection-commands";
import { QueryLanguage, tryGetQueryLanguage } from "../common/query-language";
import { LanguageContextStore } from "../language-context-store";
enum SortOrder {
NameAsc = "NameAsc",
@@ -73,7 +75,10 @@ class DatabaseTreeDataProvider
);
private currentDatabaseItem: DatabaseItem | undefined;
constructor(private databaseManager: DatabaseManager) {
constructor(
private databaseManager: DatabaseManager,
private languageContext: LanguageContextStore,
) {
super();
this.currentDatabaseItem = databaseManager.currentDatabaseItem;
@@ -88,6 +93,11 @@ class DatabaseTreeDataProvider
this.handleDidChangeCurrentDatabaseItem.bind(this),
),
);
this.push(
this.languageContext.onLanguageContextChanged(async () => {
this._onDidChangeTreeData.fire(undefined);
}),
);
}
public get onDidChangeTreeData(): Event<DatabaseItem | undefined> {
@@ -131,7 +141,15 @@ class DatabaseTreeDataProvider
public getChildren(element?: DatabaseItem): ProviderResult<DatabaseItem[]> {
if (element === undefined) {
return this.databaseManager.databaseItems.slice(0).sort((db1, db2) => {
// Filter items by language
const displayItems = this.databaseManager.databaseItems.filter((item) => {
return this.languageContext.shouldInclude(
tryGetQueryLanguage(item.language),
);
});
// Sort items
return displayItems.slice(0).sort((db1, db2) => {
switch (this.sortOrder) {
case SortOrder.NameAsc:
return db1.name.localeCompare(db2.name, env.language);
@@ -200,6 +218,7 @@ export class DatabaseUI extends DisposableObject {
public constructor(
private app: App,
private databaseManager: DatabaseManager,
private languageContext: LanguageContextStore,
private readonly queryServer: QueryRunner | undefined,
private readonly storagePath: string,
readonly extensionPath: string,
@@ -207,7 +226,7 @@ export class DatabaseUI extends DisposableObject {
super();
this.treeDataProvider = this.push(
new DatabaseTreeDataProvider(databaseManager),
new DatabaseTreeDataProvider(databaseManager, languageContext),
);
this.push(
window.createTreeView("codeQLDatabases", {
@@ -245,6 +264,60 @@ export class DatabaseUI extends DisposableObject {
this.handleMakeCurrentDatabase.bind(this),
"codeQLDatabases.sortByName": this.handleSortByName.bind(this),
"codeQLDatabases.sortByDateAdded": this.handleSortByDateAdded.bind(this),
"codeQLDatabases.displayAllLanguages":
this.handleClearLanguageFilter.bind(this),
"codeQLDatabases.displayCpp": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Cpp,
),
"codeQLDatabases.displayCsharp": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.CSharp,
),
"codeQLDatabases.displayGo": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Go,
),
"codeQLDatabases.displayJava": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Java,
),
"codeQLDatabases.displayJavascript": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Javascript,
),
"codeQLDatabases.displayPython": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Python,
),
"codeQLDatabases.displayRuby": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Ruby,
),
"codeQLDatabases.displaySwift": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Swift,
),
"codeQLDatabases.displayAllLanguagesSelected":
this.handleClearLanguageFilter.bind(this),
"codeQLDatabases.displayCppSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Cpp),
"codeQLDatabases.displayCsharpSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.CSharp),
"codeQLDatabases.displayGoSelected": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Go,
),
"codeQLDatabases.displayJavaSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Java),
"codeQLDatabases.displayJavascriptSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Javascript),
"codeQLDatabases.displayPythonSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Python),
"codeQLDatabases.displayRubySelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Ruby),
"codeQLDatabases.displaySwiftSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Swift),
"codeQLDatabases.removeDatabase": createMultiSelectionCommand(
this.handleRemoveDatabase.bind(this),
),
@@ -535,6 +608,14 @@ export class DatabaseUI extends DisposableObject {
}
}
private async handleClearLanguageFilter() {
await this.languageContext.clearLanguageContext();
}
private async handleChangeLanguageFilter(languageFilter: QueryLanguage) {
await this.languageContext.setLanguageContext(languageFilter);
}
private async handleUpgradeCurrentDatabase(): Promise<void> {
return withProgress(
async (progress, token) => {

2
extensions/ql-vscode/src/databases/ui/db-panel.ts
View File

@@ -409,7 +409,7 @@ export class DbPanel extends DisposableObject {
return;
}
void window.withProgress(
await window.withProgress(
{
location: ProgressLocation.Notification,
title: "Searching for repositories... This might take a while",

15
extensions/ql-vscode/src/extension.ts
View File

@@ -135,6 +135,7 @@ import { TestManagerBase } from "./query-testing/test-manager-base";
import { NewQueryRunner, QueryRunner, QueryServerClient } from "./query-server";
import { QueriesModule } from "./queries-panel/queries-module";
import { OpenReferencedFileCodeLensProvider } from "./local-queries/open-referenced-file-code-lens-provider";
import { LanguageContextStore } from "./language-context-store";
/**
* extension.ts
@@ -299,12 +300,12 @@ const shouldUpdateOnNextActivationKey = "shouldUpdateOnNextActivation";
const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
// This is the minimum version of vscode that we _want_ to support. We want to update the language server library, but that
// requires 1.67 or later. If we change the minimum version in the package.json, then anyone on an older version of vscode
// This is the minimum version of vscode that we _want_ to support. We want to update to Node 18, but that
// requires 1.82 or later. If we change the minimum version in the package.json, then anyone on an older version of vscode will
// silently be unable to upgrade. So, the solution is to first bump the minimum version here and release. Then
// bump the version in the package.json and release again. This way, anyone on an older version of vscode will get a warning
// before silently being refused to upgrade.
const MIN_VERSION = "1.67.0";
const MIN_VERSION = "1.82.0";
/**
* Returns the CodeQLExtensionInterface, or an empty object if the interface is not
@@ -774,17 +775,22 @@ async function activateWithInstalledDistribution(
void dbm.loadPersistedState();
ctx.subscriptions.push(dbm);
void extLogger.log("Initializing language context.");
const languageContext = new LanguageContextStore(app);
void extLogger.log("Initializing database panel.");
const databaseUI = new DatabaseUI(
app,
dbm,
languageContext,
qs,
getContextStoragePath(ctx),
ctx.extensionPath,
);
ctx.subscriptions.push(databaseUI);
QueriesModule.initialize(app, cliServer);
QueriesModule.initialize(app, languageContext, cliServer);
void extLogger.log("Initializing evaluator log viewer.");
const evalLogViewer = new EvalLogViewer();
@@ -865,6 +871,7 @@ async function activateWithInstalledDistribution(
ctx,
queryHistoryConfigurationListener,
labelProvider,
languageContext,
async (
from: CompletedLocalQueryInfo,
to: CompletedLocalQueryInfo,

49
extensions/ql-vscode/src/language-context-store.ts Normal file
View File

@@ -0,0 +1,49 @@
import { App } from "./common/app";
import { DisposableObject } from "./common/disposable-object";
import { AppEvent, AppEventEmitter } from "./common/events";
import { QueryLanguage } from "./common/query-language";
type LanguageFilter = QueryLanguage | "All";
export class LanguageContextStore extends DisposableObject {
public readonly onLanguageContextChanged: AppEvent<void>;
private readonly onLanguageContextChangedEmitter: AppEventEmitter<void>;
private languageFilter: LanguageFilter;
constructor(private readonly app: App) {
super();
// State initialization
this.languageFilter = "All";
// Set up event emitters
this.onLanguageContextChangedEmitter = this.push(
app.createEventEmitter<void>(),
);
this.onLanguageContextChanged = this.onLanguageContextChangedEmitter.event;
}
public async clearLanguageContext() {
this.languageFilter = "All";
this.onLanguageContextChangedEmitter.fire();
await this.app.commands.execute(
"setContext",
"codeQLDatabases.languageFilter",
"",
);
}
public async setLanguageContext(language: QueryLanguage) {
this.languageFilter = language;
this.onLanguageContextChangedEmitter.fire();
await this.app.commands.execute(
"setContext",
"codeQLDatabases.languageFilter",
language,
);
}
public shouldInclude(language: QueryLanguage | undefined): boolean {
return this.languageFilter === "All" || this.languageFilter === language;
}
}

35
extensions/ql-vscode/src/language-support/contextual/template-provider.ts
View File

@@ -36,6 +36,7 @@ import {
import { CoreCompletedQuery, QueryRunner } from "../../query-server";
import { AstBuilder } from "../ast-viewer/ast-builder";
import { qlpackOfDatabase } from "../../local-queries";
import { MultiCancellationToken } from "../../common/vscode/multi-cancellation-token";
/**
* Runs templated CodeQL queries to find definitions in
@@ -43,6 +44,7 @@ import { qlpackOfDatabase } from "../../local-queries";
* generalize this to other custom queries, e.g. showing dataflow to
* or from a selected identifier.
*/
export class TemplateQueryDefinitionProvider implements DefinitionProvider {
private cache: CachedOperation<LocationLink[]>;
@@ -60,11 +62,11 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
async provideDefinition(
document: TextDocument,
position: Position,
_token: CancellationToken,
token: CancellationToken,
): Promise<LocationLink[]> {
const fileLinks = this.shouldUseCache()
? await this.cache.get(document.uri.toString())
: await this.getDefinitions(document.uri.toString());
? await this.cache.get(document.uri.toString(), token)
: await this.getDefinitions(document.uri.toString(), token);
const locLinks: LocationLink[] = [];
for (const link of fileLinks) {
@@ -79,9 +81,13 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
return !(isCanary() && NO_CACHE_CONTEXTUAL_QUERIES.getValue<boolean>());
}
private async getDefinitions(uriString: string): Promise<LocationLink[]> {
private async getDefinitions(
uriString: string,
token: CancellationToken,
): Promise<LocationLink[]> {
return withProgress(
async (progress, token) => {
async (progress, tokenInner) => {
const multiToken = new MultiCancellationToken(token, tokenInner);
return getLocationsForUriString(
this.cli,
this.qs,
@@ -90,7 +96,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
KeyType.DefinitionQuery,
this.queryStorageDir,
progress,
token,
multiToken,
(src, _dest) => src === uriString,
);
},
@@ -126,11 +132,11 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
document: TextDocument,
position: Position,
_context: ReferenceContext,
_token: CancellationToken,
token: CancellationToken,
): Promise<Location[]> {
const fileLinks = this.shouldUseCache()
? await this.cache.get(document.uri.toString())
: await this.getReferences(document.uri.toString());
? await this.cache.get(document.uri.toString(), token)
: await this.getReferences(document.uri.toString(), token);
const locLinks: Location[] = [];
for (const link of fileLinks) {
@@ -148,9 +154,14 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
return !(isCanary() && NO_CACHE_CONTEXTUAL_QUERIES.getValue<boolean>());
}
private async getReferences(uriString: string): Promise<FullLocationLink[]> {
private async getReferences(
uriString: string,
token: CancellationToken,
): Promise<FullLocationLink[]> {
return withProgress(
async (progress, token) => {
async (progress, tokenInner) => {
const multiToken = new MultiCancellationToken(token, tokenInner);
return getLocationsForUriString(
this.cli,
this.qs,
@@ -159,7 +170,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
KeyType.DefinitionQuery,
this.queryStorageDir,
progress,
token,
multiToken,
(src, _dest) => src === uriString,
);
},

2
extensions/ql-vscode/src/local-queries/local-queries.ts
View File

@@ -49,6 +49,7 @@ import { LocalQueryRun } from "./local-query-run";
import { createMultiSelectionCommand } from "../common/vscode/selection-commands";
import { findLanguage } from "../codeql-cli/query-language";
import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
import { tryGetQueryLanguage } from "../common/query-language";
interface DatabaseQuickPickItem extends QuickPickItem {
databaseItem: DatabaseItem;
@@ -364,6 +365,7 @@ export class LocalQueries extends DisposableObject {
const initialInfo = await createInitialQueryInfo(selectedQuery, {
databaseUri: dbItem.databaseUri.toString(),
name: dbItem.name,
language: tryGetQueryLanguage(dbItem.language),
});
// When cancellation is requested from the query history view, we just stop the debug session.

20
extensions/ql-vscode/src/local-queries/query-resolver.ts
View File

@@ -13,6 +13,7 @@ import { redactableError } from "../common/errors";
import { showAndLogExceptionWithTelemetry } from "../common/logging";
import { extLogger } from "../common/logging/vscode";
import { telemetryListener } from "../common/vscode/telemetry";
import { SuiteInstruction } from "../packaging/suite-instruction";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
@@ -38,24 +39,26 @@ export interface QueryConstraints {
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param constraints Constraints on the queries to search for.
* @param additionalPacks Additional pack paths to search.
* @returns The found queries from the first pack in which any matching queries were found.
*/
async function resolveQueriesFromPacks(
export async function resolveQueriesFromPacks(
cli: CodeQLCliServer,
qlpacks: string[],
constraints: QueryConstraints,
additionalPacks: string[] = [],
): Promise<string[]> {
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
const suiteYaml: SuiteInstruction[] = [];
for (const qlpack of qlpacks) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: constraints,
include: constraints as Record<string, string[]>,
});
}
await writeFile(
@@ -66,10 +69,10 @@ async function resolveQueriesFromPacks(
"utf8",
);
return await cli.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
return await cli.resolveQueriesInSuite(suiteFile, [
...getOnDiskWorkspaceFolders(),
...additionalPacks,
]);
}
export async function resolveQueriesByLanguagePack(
@@ -96,6 +99,7 @@ export async function resolveQueriesByLanguagePack(
* @param packsToSearch The list of packs to search.
* @param name The name of the query to use in error messages.
* @param constraints Constraints on the queries to search for.
* @param additionalPacks Additional pack paths to search.
* @returns The found queries from the first pack in which any matching queries were found.
*/
export async function resolveQueries(
@@ -103,11 +107,13 @@ export async function resolveQueries(
packsToSearch: string[],
name: string,
constraints: QueryConstraints,
additionalPacks: string[] = [],
): Promise<string[]> {
const queries = await resolveQueriesFromPacks(
cli,
packsToSearch,
constraints,
additionalPacks,
);
if (queries.length > 0) {
return queries;

26
extensions/ql-vscode/src/local-queries/results-view.ts
View File

@@ -75,6 +75,7 @@ import { telemetryListener } from "../common/vscode/telemetry";
import { redactableError } from "../common/errors";
import { ResultsViewCommands } from "../common/commands";
import { App } from "../common/app";
import { Disposable } from "../common/disposable-object";
/**
* results-view.ts
@@ -157,6 +158,12 @@ function numInterpretedPages(
return Math.ceil(n / pageSize);
}
/**
* The results view is used for displaying the results of a local query. It is a singleton; only 1 results view exists
* in the extension. It is created when the extension is activated and disposed of when the extension is deactivated.
* There can be multiple panels linked to this view over the lifetime of the extension, but there is only ever 1 panel
* active at a time.
*/
export class ResultsView extends AbstractWebview<
IntoResultsViewMsg,
FromResultsViewMsg
@@ -168,6 +175,9 @@ export class ResultsView extends AbstractWebview<
"codeql-query-results",
);
// Event listeners that should be disposed of when the view is disposed.
private disposableEventListeners: Disposable[] = [];
constructor(
app: App,
private databaseManager: DatabaseManager,
@@ -176,14 +186,16 @@ export class ResultsView extends AbstractWebview<
private labelProvider: HistoryItemLabelProvider,
) {
super(app);
this.push(this._diagnosticCollection);
this.push(
// We can't use this.push for these two event listeners because they need to be disposed of when the view is
// disposed, not when the panel is disposed. The results view is a singleton, so we shouldn't be calling this.push.
this.disposableEventListeners.push(
vscode.window.onDidChangeTextEditorSelection(
this.handleSelectionChange.bind(this),
),
);
this.push(
this.disposableEventListeners.push(
this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
if (kind === DatabaseEventKind.Remove) {
this._diagnosticCollection.clear();
@@ -981,4 +993,12 @@ export class ResultsView extends AbstractWebview<
editor.setDecorations(shownLocationLineDecoration, []);
}
}
dispose() {
super.dispose();
this._diagnosticCollection.dispose();
this.disposableEventListeners.forEach((d) => d.dispose());
this.disposableEventListeners = [];
}
}

104
extensions/ql-vscode/src/log-insights/summary-parser.ts
View File

@@ -1,4 +1,5 @@
import { writeFile, promises } from "fs-extra";
import { createReadStream, writeFile } from "fs-extra";
import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
/**
* Location information for a single pipeline invocation in the RA.
@@ -64,59 +65,64 @@ export async function generateSummarySymbolsFile(
async function generateSummarySymbols(
summaryPath: string,
): Promise<SummarySymbols> {
const summary = await promises.readFile(summaryPath, {
const stream = createReadStream(summaryPath, {
encoding: "utf-8",
});
const symbols: SummarySymbols = {
predicates: {},
};
try {
const lines = splitStreamAtSeparators(stream, LINE_ENDINGS);
const lines = summary.split(/\r?\n/);
let lineNumber = 0;
while (lineNumber < lines.length) {
const startLineNumber = lineNumber;
lineNumber++;
const startLine = lines[startLineNumber];
const nonRecursiveMatch = startLine.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
let predicateName: string | undefined = undefined;
const symbols: SummarySymbols = {
predicates: {},
};
let lineNumber = 0;
let raStartLine = 0;
let iteration = 0;
if (nonRecursiveMatch) {
predicateName = nonRecursiveMatch.groups!.predicateName;
} else {
const recursiveMatch = startLine.match(RECURSIVE_TUPLE_COUNT_REGEXP);
if (recursiveMatch?.groups) {
predicateName = recursiveMatch.groups.predicateName;
iteration = parseInt(recursiveMatch.groups.iteration);
}
}
if (predicateName !== undefined) {
const raStartLine = lineNumber;
let raEndLine: number | undefined = undefined;
while (lineNumber < lines.length && raEndLine === undefined) {
const raLine = lines[lineNumber];
const returnMatch = raLine.match(RETURN_REGEXP);
let predicateName: string | undefined = undefined;
let startLine = 0;
for await (const line of lines) {
if (predicateName === undefined) {
// Looking for the start of the predicate.
const nonRecursiveMatch = line.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
if (nonRecursiveMatch) {
iteration = 0;
predicateName = nonRecursiveMatch.groups!.predicateName;
} else {
const recursiveMatch = line.match(RECURSIVE_TUPLE_COUNT_REGEXP);
if (recursiveMatch?.groups) {
predicateName = recursiveMatch.groups.predicateName;
iteration = parseInt(recursiveMatch.groups.iteration);
}
}
if (predicateName !== undefined) {
startLine = lineNumber;
raStartLine = lineNumber + 1;
}
} else {
const returnMatch = line.match(RETURN_REGEXP);
if (returnMatch) {
raEndLine = lineNumber;
}
lineNumber++;
}
if (raEndLine !== undefined) {
let symbol = symbols.predicates[predicateName];
if (symbol === undefined) {
symbol = {
iterations: {},
let symbol = symbols.predicates[predicateName];
if (symbol === undefined) {
symbol = {
iterations: {},
};
symbols.predicates[predicateName] = symbol;
}
symbol.iterations[iteration] = {
startLine,
raStartLine,
raEndLine: lineNumber,
};
symbols.predicates[predicateName] = symbol;
}
symbol.iterations[iteration] = {
startLine: lineNumber,
raStartLine,
raEndLine,
};
}
}
}
return symbols;
predicateName = undefined;
}
}
lineNumber++;
}
return symbols;
} finally {
stream.close();
}
}

13
extensions/ql-vscode/src/model-editor/auto-model-codeml-queries.ts
View File

@@ -7,7 +7,6 @@ import { Mode } from "./shared/mode";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { interpretResultsSarif } from "../query-results";
import { join } from "path";
import { assertNever } from "../common/helpers-pure";
import { dir } from "tmp-promise";
import { writeFile, outputFile } from "fs-extra";
import { dump as dumpYaml } from "js-yaml";
@@ -16,17 +15,7 @@ import { runQuery } from "../local-queries/run-query";
import { QueryMetadata } from "../common/interface-types";
import { CancellationTokenSource } from "vscode";
import { resolveQueries } from "../local-queries";
function modeTag(mode: Mode): string {
switch (mode) {
case Mode.Application:
return "application-mode";
case Mode.Framework:
return "framework-mode";
default:
assertNever(mode);
}
}
import { modeTag } from "./mode-tag";
type AutoModelQueriesOptions = {
mode: Mode;

7
extensions/ql-vscode/src/model-editor/auto-modeler.ts
View File

@@ -16,6 +16,7 @@ import { QueryRunner } from "../query-server";
import { DatabaseItem } from "../databases/local-databases";
import { Mode } from "./shared/mode";
import { CancellationTokenSource } from "vscode";
import { convertToLegacyModeledMethods } from "./modeled-methods-legacy";
// Limit the number of candidates we send to the model in each request
// to avoid long requests.
@@ -192,11 +193,13 @@ export class AutoModeler {
filename: "auto-model.yml",
});
const loadedMethods = loadDataExtensionYaml(models);
if (!loadedMethods) {
const rawLoadedMethods = loadDataExtensionYaml(models);
if (!rawLoadedMethods) {
return;
}
const loadedMethods = convertToLegacyModeledMethods(rawLoadedMethods);
// Any candidate that was part of the response is a negative result
// meaning that the canidate is not a sink for the kinds that the LLM is checking for.
// For now we model this as a sink neutral method, however this is subject

72
extensions/ql-vscode/src/model-editor/bqrs.ts
View File

@@ -2,41 +2,65 @@ import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
import { Call, CallClassification, Method } from "./method";
import { ModeledMethodType } from "./modeled-method";
import { parseLibraryFilename } from "./library";
import { Mode } from "./shared/mode";
import { ApplicationModeTuple, FrameworkModeTuple } from "./queries/query";
export function decodeBqrsToMethods(chunk: DecodedBqrsChunk): Method[] {
export function decodeBqrsToMethods(
chunk: DecodedBqrsChunk,
mode: Mode,
): Method[] {
const methodsByApiName = new Map<string, Method>();
chunk?.tuples.forEach((tuple) => {
const usage = tuple[0] as Call;
const signature = tuple[1] as string;
const supported = (tuple[2] as string) === "true";
let library = tuple[4] as string;
let libraryVersion: string | undefined = tuple[5] as string;
const type = tuple[6] as ModeledMethodType;
const classification = tuple[8] as CallClassification;
let usage: Call;
let packageName: string;
let typeName: string;
let methodName: string;
let methodParameters: string;
let supported: boolean;
let library: string;
let libraryVersion: string | undefined;
let type: ModeledMethodType;
let classification: CallClassification;
const [packageWithType, methodDeclaration] = signature.split("#");
if (mode === Mode.Application) {
[
usage,
packageName,
typeName,
methodName,
methodParameters,
supported,
library,
libraryVersion,
type,
classification,
] = tuple as ApplicationModeTuple;
} else {
[
usage,
packageName,
typeName,
methodName,
methodParameters,
supported,
library,
type,
] = tuple as FrameworkModeTuple;
const packageName = packageWithType.substring(
0,
packageWithType.lastIndexOf("."),
);
const typeName = packageWithType.substring(
packageWithType.lastIndexOf(".") + 1,
);
classification = CallClassification.Unknown;
}
const methodName = methodDeclaration.substring(
0,
methodDeclaration.indexOf("("),
);
const methodParameters = methodDeclaration.substring(
methodDeclaration.indexOf("("),
);
const signature = `${packageName}.${typeName}#${methodName}${methodParameters}`;
// For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
// properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
// for Java.
if (library.endsWith(".jar") || libraryVersion === "") {
if (
library.endsWith(".jar") ||
libraryVersion === "" ||
libraryVersion === undefined
) {
const { name, version } = parseLibraryFilename(library);
library = name;
if (version) {

45
extensions/ql-vscode/src/model-editor/data-schema.json
View File

@@ -1,45 +0,0 @@
{
"type": "object",
"properties": {
"extensions": {
"type": "array",
"items": {
"type": "object",
"required": ["addsTo", "data"],
"properties": {
"addsTo": {
"type": "object",
"required": ["pack", "extensible"],
"properties": {
"pack": {
"type": "string"
},
"extensible": {
"type": "string"
}
}
},
"data": {
"type": "array",
"items": {
"type": "array",
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "boolean"
},
{
"type": "number"
}
]
}
}
}
}
}
}
}
}

97
extensions/ql-vscode/src/model-editor/extension-pack-metadata.schema.json Normal file
View File

@@ -0,0 +1,97 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/ExtensionPackMetadata",
"definitions": {
"ExtensionPackMetadata": {
"type": "object",
"properties": {
"extensionTargets": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"dataExtensions": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
}
]
},
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"dependencies": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"dbscheme": {
"type": "string"
},
"library": {
"type": "boolean"
},
"defaultSuite": {
"type": "array",
"items": {
"$ref": "#/definitions/SuiteInstruction"
}
},
"defaultSuiteFile": {
"type": "string"
}
},
"required": ["dataExtensions", "extensionTargets", "name", "version"]
},
"SuiteInstruction": {
"type": "object",
"properties": {
"qlpack": {
"type": "string"
},
"query": {
"type": "string"
},
"queries": {
"type": "string"
},
"include": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"exclude": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"from": {
"type": "string"
}
},
"description": "A single entry in a .qls file."
}
}
}

7
extensions/ql-vscode/src/model-editor/extension-pack-metadata.ts Normal file
View File

@@ -0,0 +1,7 @@
import { QlPackFile } from "../packaging/qlpack-file";
export type ExtensionPackMetadata = QlPackFile & {
// Make both extensionTargets and dataExtensions required
extensionTargets: Record<string, string>;
dataExtensions: string[] | string;
};

27
extensions/ql-vscode/src/model-editor/extension-pack-picker.ts
View File

@@ -2,6 +2,7 @@ import { join } from "path";
import { outputFile, pathExists, readFile } from "fs-extra";
import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import { Uri } from "vscode";
import Ajv from "ajv";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { ProgressCallback } from "../common/vscode/progress";
@@ -18,6 +19,12 @@ import {
} from "./extension-pack-name";
import { autoPickExtensionsDirectory } from "./extensions-workspace-folder";
import { ExtensionPackMetadata } from "./extension-pack-metadata";
import * as extensionPackMetadataSchemaJson from "./extension-pack-metadata.schema.json";
const ajv = new Ajv({ allErrors: true });
const extensionPackValidate = ajv.compile(extensionPackMetadataSchemaJson);
export async function pickExtensionPack(
cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
databaseItem: Pick<DatabaseItem, "name" | "language">,
@@ -170,6 +177,22 @@ async function writeExtensionPack(
return extensionPack;
}
function validateExtensionPack(
extensionPack: unknown,
): extensionPack is ExtensionPackMetadata {
extensionPackValidate(extensionPack);
if (extensionPackValidate.errors) {
throw new Error(
`Invalid extension pack YAML: ${extensionPackValidate.errors
.map((error) => `${error.instancePath} ${error.message}`)
.join(", ")}`,
);
}
return true;
}
async function readExtensionPack(
path: string,
language: string,
@@ -188,6 +211,10 @@ async function readExtensionPack(
throw new Error(`Could not parse ${qlpackPath}`);
}
if (!validateExtensionPack(qlpack)) {
throw new Error(`Could not validate ${qlpackPath}`);
}
const dataExtensionValue = qlpack.dataExtensions;
if (
!(

33
extensions/ql-vscode/src/model-editor/external-api-usage-queries.ts
View File

@@ -16,6 +16,10 @@ import { fetchExternalApiQueries } from "./queries";
import { Method } from "./method";
import { runQuery } from "../local-queries/run-query";
import { decodeBqrsToMethods } from "./bqrs";
import {
resolveEndpointsQuery,
syntheticQueryPackName,
} from "./model-editor-queries";
type RunQueryOptions = {
cliServer: CodeQLCliServer;
@@ -88,7 +92,28 @@ export async function runExternalApiQueries(
await cliServer.resolveQlpacks(additionalPacks, true),
);
const queryPath = join(queryDir, queryNameFromMode(mode));
progress({
message: "Resolving query",
step: 2,
maxStep: externalApiQueriesProgressMaxStep,
});
// Resolve the queries from either codeql/java-queries or from the temporary queryDir
const queryPath = await resolveEndpointsQuery(
cliServer,
databaseItem.language,
mode,
[syntheticQueryPackName],
[queryDir],
);
if (!queryPath) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError`The ${mode} model editor query could not be found. Try re-opening the model editor. If that doesn't work, try upgrading the CodeQL libraries.`,
);
return;
}
// Run the actual query
const completedQuery = await runQuery({
@@ -132,7 +157,7 @@ export async function runExternalApiQueries(
maxStep: externalApiQueriesProgressMaxStep,
});
return decodeBqrsToMethods(bqrsChunk);
return decodeBqrsToMethods(bqrsChunk, mode);
}
type GetResultsOptions = {
@@ -160,7 +185,5 @@ export async function readQueryResults({
}
function queryNameFromMode(mode: Mode): string {
return `FetchExternalApis${
mode.charAt(0).toUpperCase() + mode.slice(1)
}Mode.ql`;
return `${mode.charAt(0).toUpperCase() + mode.slice(1)}ModeEndpoints.ql`;
}

14
extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts
View File

@@ -3,14 +3,24 @@ import { App } from "../../common/app";
import { DisposableObject } from "../../common/disposable-object";
import { MethodModelingViewProvider } from "./method-modeling-view-provider";
import { Method } from "../method";
import { ModelingStore } from "../modeling-store";
import { ModelEditorViewTracker } from "../model-editor-view-tracker";
export class MethodModelingPanel extends DisposableObject {
private readonly provider: MethodModelingViewProvider;
constructor(app: App) {
constructor(
app: App,
modelingStore: ModelingStore,
editorViewTracker: ModelEditorViewTracker,
) {
super();
this.provider = new MethodModelingViewProvider(app);
this.provider = new MethodModelingViewProvider(
app,
modelingStore,
editorViewTracker,
);
this.push(
window.registerWebviewViewProvider(
MethodModelingViewProvider.viewType,

167
extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts
View File

@@ -1,67 +1,74 @@
import * as vscode from "vscode";
import { Uri, WebviewViewProvider } from "vscode";
import { getHtmlForWebview } from "../../common/vscode/webview-html";
import { FromMethodModelingMessage } from "../../common/interface-types";
import {
FromMethodModelingMessage,
ToMethodModelingMessage,
} from "../../common/interface-types";
import { telemetryListener } from "../../common/vscode/telemetry";
import { showAndLogExceptionWithTelemetry } from "../../common/logging/notifications";
import { extLogger } from "../../common/logging/vscode/loggers";
import { App } from "../../common/app";
import { redactableError } from "../../common/errors";
import { Method } from "../method";
import { DbModelingState, ModelingStore } from "../modeling-store";
import { AbstractWebviewViewProvider } from "../../common/vscode/abstract-webview-view-provider";
import { assertNever } from "../../common/helpers-pure";
import { ModelEditorViewTracker } from "../model-editor-view-tracker";
export class MethodModelingViewProvider implements WebviewViewProvider {
export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
ToMethodModelingMessage,
FromMethodModelingMessage
> {
public static readonly viewType = "codeQLMethodModeling";
private webviewView: vscode.WebviewView | undefined = undefined;
private method: Method | undefined = undefined;
constructor(private readonly app: App) {}
/**
* This is called when a view first becomes visible. This may happen when the view is
* first loaded or when the user hides and then shows a view again.
*/
public resolveWebviewView(
webviewView: vscode.WebviewView,
_context: vscode.WebviewViewResolveContext,
_token: vscode.CancellationToken,
constructor(
app: App,
private readonly modelingStore: ModelingStore,
private readonly editorViewTracker: ModelEditorViewTracker,
) {
webviewView.webview.options = {
enableScripts: true,
localResourceRoots: [Uri.file(this.app.extensionPath)],
};
super(app, "method-modeling");
}
const html = getHtmlForWebview(
this.app,
webviewView.webview,
"method-modeling",
{
allowInlineStyles: true,
allowWasmEval: false,
},
);
webviewView.webview.html = html;
webviewView.webview.onDidReceiveMessage(async (msg) => this.onMessage(msg));
this.webviewView = webviewView;
protected override onWebViewLoaded(): void {
this.setInitialState();
this.registerToModelingStoreEvents();
}
public async setMethod(method: Method): Promise<void> {
if (this.webviewView) {
await this.webviewView.webview.postMessage({
this.method = method;
if (this.isShowingView) {
await this.postMessage({
t: "setMethod",
method,
});
}
}
private async onMessage(msg: FromMethodModelingMessage): Promise<void> {
private setInitialState(): void {
const selectedMethod = this.modelingStore.getSelectedMethodDetails();
if (selectedMethod) {
void this.postMessage({
t: "setSelectedMethod",
method: selectedMethod.method,
modeledMethod: selectedMethod.modeledMethod,
isModified: selectedMethod.isModified,
});
}
}
protected override async onMessage(
msg: FromMethodModelingMessage,
): Promise<void> {
switch (msg.t) {
case "telemetry": {
case "viewLoaded":
this.onWebViewLoaded();
break;
case "telemetry":
telemetryListener?.sendUIInteraction(msg.action);
break;
}
case "unhandledError":
void showAndLogExceptionWithTelemetry(
extLogger,
@@ -71,6 +78,86 @@ export class MethodModelingViewProvider implements WebviewViewProvider {
)`Unhandled error in method modeling view: ${msg.error.message}`,
);
break;
case "setModeledMethod": {
const activeState = this.ensureActiveState();
this.modelingStore.updateModeledMethod(
activeState.databaseItem,
msg.method,
);
break;
}
case "revealInModelEditor":
await this.revealInModelEditor(msg.method);
break;
default:
assertNever(msg);
}
}
private async revealInModelEditor(method: Method): Promise<void> {
const activeState = this.ensureActiveState();
const views = this.editorViewTracker.getViews(
activeState.databaseItem.databaseUri.toString(),
);
if (views.length === 0) {
return;
}
await Promise.all(views.map((view) => view.revealMethod(method)));
}
private ensureActiveState(): DbModelingState {
const activeState = this.modelingStore.getStateForActiveDb();
if (!activeState) {
throw new Error("No active state found in modeling store");
}
return activeState;
}
private registerToModelingStoreEvents(): void {
this.push(
this.modelingStore.onModeledMethodsChanged(async (e) => {
if (this.webviewView && e.isActiveDb) {
const modeledMethod = e.modeledMethods[this.method?.signature ?? ""];
if (modeledMethod) {
await this.postMessage({
t: "setModeledMethod",
method: modeledMethod,
});
}
}
}),
);
this.push(
this.modelingStore.onModifiedMethodsChanged(async (e) => {
if (this.webviewView && e.isActiveDb && this.method) {
const isModified = e.modifiedMethods.has(this.method.signature);
await this.postMessage({
t: "setMethodModified",
isModified,
});
}
}),
);
this.push(
this.modelingStore.onSelectedMethodChanged(async (e) => {
if (this.webviewView) {
this.method = e.method;
await this.postMessage({
t: "setSelectedMethod",
method: e.method,
modeledMethod: e.modeledMethod,
isModified: e.isModified,
});
}
}),
);
}
}

8
extensions/ql-vscode/src/model-editor/method.ts
View File

@@ -57,3 +57,11 @@ export interface Method extends MethodSignature {
supportedType: ModeledMethodType;
usages: Usage[];
}
export function getArgumentsList(methodParameters: string): string[] {
if (methodParameters === "()") {
return [];
}
return methodParameters.substring(1, methodParameters.length - 1).split(",");
}

36
extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-data-provider.ts
View File

@@ -14,6 +14,9 @@ import { DatabaseItem } from "../../databases/local-databases";
import { relative } from "path";
import { CodeQLCliServer } from "../../codeql-cli/cli";
import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "../shared/hide-modeled-methods";
import { getModelingStatus } from "../shared/modeling-status";
import { assertNever } from "../../common/helpers-pure";
import { ModeledMethod } from "../modeled-method";
export class MethodsUsageDataProvider
extends DisposableObject
@@ -23,6 +26,8 @@ export class MethodsUsageDataProvider
private databaseItem: DatabaseItem | undefined = undefined;
private sourceLocationPrefix: string | undefined = undefined;
private hideModeledMethods: boolean = INITIAL_HIDE_MODELED_METHODS_VALUE;
private modeledMethods: Record<string, ModeledMethod> = {};
private modifiedMethodSignatures: Set<string> = new Set();
private readonly onDidChangeTreeDataEmitter = this.push(
new EventEmitter<void>(),
@@ -47,17 +52,23 @@ export class MethodsUsageDataProvider
methods: Method[],
databaseItem: DatabaseItem,
hideModeledMethods: boolean,
modeledMethods: Record<string, ModeledMethod>,
modifiedMethodSignatures: Set<string>,
): Promise<void> {
if (
this.methods !== methods ||
this.databaseItem !== databaseItem ||
this.hideModeledMethods !== hideModeledMethods
this.hideModeledMethods !== hideModeledMethods ||
this.modeledMethods !== modeledMethods ||
this.modifiedMethodSignatures !== modifiedMethodSignatures
) {
this.methods = methods;
this.databaseItem = databaseItem;
this.sourceLocationPrefix =
await this.databaseItem.getSourceLocationPrefix(this.cliServer);
this.hideModeledMethods = hideModeledMethods;
this.modeledMethods = modeledMethods;
this.modifiedMethodSignatures = modifiedMethodSignatures;
this.onDidChangeTreeDataEmitter.fire();
}
@@ -68,7 +79,7 @@ export class MethodsUsageDataProvider
return {
label: `${item.packageName}.${item.typeName}.${item.methodName}${item.methodParameters}`,
collapsibleState: TreeItemCollapsibleState.Collapsed,
iconPath: new ThemeIcon("symbol-method"),
iconPath: this.getModelingStatusIcon(item),
};
} else {
const method = this.getParent(item);
@@ -83,11 +94,30 @@ export class MethodsUsageDataProvider
command: "codeQLModelEditor.jumpToUsageLocation",
arguments: [method, item, this.databaseItem],
},
iconPath: new ThemeIcon("error", new ThemeColor("errorForeground")),
};
}
}
private getModelingStatusIcon(method: Method): ThemeIcon {
const modeledMethod = this.modeledMethods[method.signature];
const modifiedMethod = this.modifiedMethodSignatures.has(method.signature);
const status = getModelingStatus(modeledMethod, modifiedMethod);
switch (status) {
case "unmodeled":
return new ThemeIcon("error", new ThemeColor("errorForeground"));
case "unsaved":
return new ThemeIcon("pass", new ThemeColor("testing.iconPassed"));
case "saved":
return new ThemeIcon(
"pass-filled",
new ThemeColor("testing.iconPassed"),
);
default:
assertNever(status);
}
}
private relativePathWithinDatabase(uri: string): string {
const parsedUri = Uri.parse(uri);
if (this.sourceLocationPrefix) {

64
extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-panel.ts
View File

@@ -7,12 +7,17 @@ import {
import { Method, Usage } from "../method";
import { DatabaseItem } from "../../databases/local-databases";
import { CodeQLCliServer } from "../../codeql-cli/cli";
import { ModelingStore } from "../modeling-store";
import { ModeledMethod } from "../modeled-method";
export class MethodsUsagePanel extends DisposableObject {
private readonly dataProvider: MethodsUsageDataProvider;
private readonly treeView: TreeView<MethodsUsageTreeViewItem>;
public constructor(cliServer: CodeQLCliServer) {
public constructor(
private readonly modelingStore: ModelingStore,
cliServer: CodeQLCliServer,
) {
super();
this.dataProvider = new MethodsUsageDataProvider(cliServer);
@@ -21,14 +26,24 @@ export class MethodsUsagePanel extends DisposableObject {
treeDataProvider: this.dataProvider,
});
this.push(this.treeView);
this.registerToModelingStoreEvents();
}
public async setState(
methods: Method[],
databaseItem: DatabaseItem,
hideModeledMethods: boolean,
modeledMethods: Record<string, ModeledMethod>,
modifiedMethodSignatures: Set<string>,
): Promise<void> {
await this.dataProvider.setState(methods, databaseItem, hideModeledMethods);
await this.dataProvider.setState(
methods,
databaseItem,
hideModeledMethods,
modeledMethods,
modifiedMethodSignatures,
);
const numOfApis = hideModeledMethods
? methods.filter((api) => !api.supported).length
: methods.length;
@@ -44,4 +59,49 @@ export class MethodsUsagePanel extends DisposableObject {
await this.treeView.reveal(canonicalUsage);
}
}
private registerToModelingStoreEvents(): void {
this.push(
this.modelingStore.onActiveDbChanged(async () => {
await this.handleStateChangeEvent();
}),
);
this.push(
this.modelingStore.onMethodsChanged(async (event) => {
if (event.isActiveDb) {
await this.handleStateChangeEvent();
}
}),
);
this.push(
this.modelingStore.onHideModeledMethodsChanged(async (event) => {
if (event.isActiveDb) {
await this.handleStateChangeEvent();
}
}),
);
this.push(
this.modelingStore.onModifiedMethodsChanged(async (event) => {
if (event.isActiveDb) {
await this.handleStateChangeEvent();
}
}),
);
}
private async handleStateChangeEvent(): Promise<void> {
const activeState = this.modelingStore.getStateForActiveDb();
if (activeState !== undefined) {
await this.setState(
activeState.methods,
activeState.databaseItem,
activeState.hideModeledMethods,
activeState.modeledMethods,
activeState.modifiedMethodSignatures,
);
}
}
}

13
extensions/ql-vscode/src/model-editor/mode-tag.ts Normal file
View File

@@ -0,0 +1,13 @@
import { Mode } from "./shared/mode";
import { assertNever } from "../common/helpers-pure";
export function modeTag(mode: Mode): string {
switch (mode) {
case Mode.Application:
return "application-mode";
case Mode.Framework:
return "framework-mode";
default:
assertNever(mode);
}
}

67
extensions/ql-vscode/src/model-editor/model-editor-module.ts
View File

@@ -15,20 +15,22 @@ import { isQueryLanguage } from "../common/query-language";
import { DisposableObject } from "../common/disposable-object";
import { MethodsUsagePanel } from "./methods-usage/methods-usage-panel";
import { Mode } from "./shared/mode";
import { showResolvableLocation } from "../databases/local-databases/locations";
import { Method, Usage } from "./method";
import { setUpPack } from "./model-editor-queries";
import { MethodModelingPanel } from "./method-modeling/method-modeling-panel";
import { ModelingStore } from "./modeling-store";
import { showResolvableLocation } from "../databases/local-databases/locations";
import { ModelEditorViewTracker } from "./model-editor-view-tracker";
const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
export class ModelEditorModule extends DisposableObject {
private readonly queryStorageDir: string;
private readonly modelingStore: ModelingStore;
private readonly editorViewTracker: ModelEditorViewTracker<ModelEditorView>;
private readonly methodsUsagePanel: MethodsUsagePanel;
private readonly methodModelingPanel: MethodModelingPanel;
private mostRecentlyActiveView: ModelEditorView | undefined = undefined;
private constructor(
private readonly app: App,
private readonly databaseManager: DatabaseManager,
@@ -38,22 +40,16 @@ export class ModelEditorModule extends DisposableObject {
) {
super();
this.queryStorageDir = join(baseQueryStorageDir, "model-editor-results");
this.methodsUsagePanel = this.push(new MethodsUsagePanel(cliServer));
this.methodModelingPanel = this.push(new MethodModelingPanel(app));
}
this.modelingStore = new ModelingStore(app);
this.editorViewTracker = new ModelEditorViewTracker();
this.methodsUsagePanel = this.push(
new MethodsUsagePanel(this.modelingStore, cliServer),
);
this.methodModelingPanel = this.push(
new MethodModelingPanel(app, this.modelingStore, this.editorViewTracker),
);
private handleViewBecameActive(view: ModelEditorView): void {
this.mostRecentlyActiveView = view;
}
private handleViewWasDisposed(view: ModelEditorView): void {
if (this.mostRecentlyActiveView === view) {
this.mostRecentlyActiveView = undefined;
}
}
private isMostRecentlyActiveView(view: ModelEditorView): boolean {
return this.mostRecentlyActiveView === view;
this.registerToModelingStoreEvents();
}
public static async initialize(
@@ -139,6 +135,7 @@ export class ModelEditorModule extends DisposableObject {
const { path: queryDir, cleanup: cleanupQueryDir } = await dir({
unsafeCleanup: true,
});
const success = await setUpPack(this.cliServer, queryDir, language);
if (!success) {
await cleanupQueryDir();
@@ -153,6 +150,8 @@ export class ModelEditorModule extends DisposableObject {
const view = new ModelEditorView(
this.app,
this.modelingStore,
this.editorViewTracker,
this.databaseManager,
this.cliServer,
this.queryRunner,
@@ -161,16 +160,14 @@ export class ModelEditorModule extends DisposableObject {
db,
modelFile,
Mode.Application,
this.methodsUsagePanel.setState.bind(this.methodsUsagePanel),
this.showMethod.bind(this),
this.handleViewBecameActive.bind(this),
(view) => {
this.handleViewWasDisposed(view);
void cleanupQueryDir();
},
this.isMostRecentlyActiveView.bind(this),
);
this.modelingStore.onDbClosed(async (dbUri) => {
if (dbUri === db.databaseUri.toString()) {
await cleanupQueryDir();
}
});
this.push(view);
this.push({
dispose(): void {
@@ -190,8 +187,7 @@ export class ModelEditorModule extends DisposableObject {
usage: Usage,
databaseItem: DatabaseItem,
) => {
await this.methodModelingPanel.setMethod(method);
await showResolvableLocation(usage.url, databaseItem, this.app.logger);
this.modelingStore.setSelectedMethod(databaseItem, method, usage);
},
};
}
@@ -200,8 +196,21 @@ export class ModelEditorModule extends DisposableObject {
await ensureDir(this.queryStorageDir);
}
private async showMethod(method: Method, usage: Usage): Promise<void> {
private registerToModelingStoreEvents(): void {
this.push(
this.modelingStore.onSelectedMethodChanged(async (event) => {
await this.showMethod(event.databaseItem, event.method, event.usage);
}),
);
}
private async showMethod(
databaseItem: DatabaseItem,
method: Method,
usage: Usage,
): Promise<void> {
await this.methodsUsagePanel.revealItem(usage);
await this.methodModelingPanel.setMethod(method);
await showResolvableLocation(usage.url, databaseItem, this.app.logger);
}
}

136
extensions/ql-vscode/src/model-editor/model-editor-queries.ts
View File

@@ -5,9 +5,27 @@ import { dump } from "js-yaml";
import { prepareExternalApiQuery } from "./external-api-usage-queries";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { showLlmGeneration } from "../config";
import { Mode } from "./shared/mode";
import { resolveQueriesFromPacks } from "../local-queries";
import { modeTag } from "./mode-tag";
export const syntheticQueryPackName = "codeql/external-api-usage";
/**
* setUpPack sets up a directory to use for the data extension editor queries.
* setUpPack sets up a directory to use for the data extension editor queries if required.
*
* There are two cases (example language is Java):
* - In case the queries are present in the codeql/java-queries, we don't need to write our own queries
* to disk. We still need to create a synthetic query pack so we can pass the queryDir to the query
* resolver without caring about whether the queries are present in the pack or not.
* - In case the queries are not present in the codeql/java-queries, we need to write our own queries
* to disk. We will create a synthetic query pack and install its dependencies so it is fully independent
* and we can simply pass it through when resolving the queries.
*
* These steps together ensure that later steps of the process don't need to keep track of whether the queries
* are present in codeql/java-queries or in our own query pack. They just need to resolve the query.
*
* @param cliServer The CodeQL CLI server to use.
* @param queryDir The directory to set up.
* @param language The language to use for the queries.
* @returns true if the setup was successful, false otherwise.
@@ -17,34 +35,104 @@ export async function setUpPack(
queryDir: string,
language: QueryLanguage,
): Promise<boolean> {
// Create the external API query
const externalApiQuerySuccess = await prepareExternalApiQuery(
queryDir,
language,
);
if (!externalApiQuerySuccess) {
return false;
}
// Set up a synthetic pack so that the query can be resolved later.
const syntheticQueryPack = {
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${language}-all`]: "*",
},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
await cliServer.packInstall(queryDir);
// Install the other needed query packs
// Download the required query packs
await cliServer.packDownload([`codeql/${language}-queries`]);
// We'll only check if the application mode query exists in the pack and assume that if it does,
// the framework mode query will also exist.
const applicationModeQuery = await resolveEndpointsQuery(
cliServer,
language,
Mode.Application,
[],
[],
);
if (applicationModeQuery) {
// Set up a synthetic pack so CodeQL doesn't crash later when we try
// to resolve a query within this directory
const syntheticQueryPack = {
name: syntheticQueryPackName,
version: "0.0.0",
dependencies: {},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
} else {
// If we can't resolve the query, we need to write them to desk ourselves.
const externalApiQuerySuccess = await prepareExternalApiQuery(
queryDir,
language,
);
if (!externalApiQuerySuccess) {
return false;
}
// Set up a synthetic pack so that the query can be resolved later.
const syntheticQueryPack = {
name: syntheticQueryPackName,
version: "0.0.0",
dependencies: {
[`codeql/${language}-all`]: "*",
},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
await cliServer.packInstall(queryDir);
}
// Download any other required packs
if (language === "java" && showLlmGeneration()) {
await cliServer.packDownload([`codeql/${language}-automodel-queries`]);
}
return true;
}
/**
* Resolve the query path to the model editor endpoints query. All queries are tagged like this:
* modeleditor endpoints <mode>
* Example: modeleditor endpoints framework-mode
*
* @param cliServer The CodeQL CLI server to use.
* @param language The language of the query pack to use.
* @param mode The mode to resolve the query for.
* @param additionalPackNames Additional pack names to search.
* @param additionalPackPaths Additional pack paths to search.
*/
export async function resolveEndpointsQuery(
cliServer: CodeQLCliServer,
language: string,
mode: Mode,
additionalPackNames: string[] = [],
additionalPackPaths: string[] = [],
): Promise<string | undefined> {
const packsToSearch = [`codeql/${language}-queries`, ...additionalPackNames];
// First, resolve the query that we want to run.
// All queries are tagged like this:
// internal extract automodel <mode> <queryTag>
// Example: internal extract automodel framework-mode candidates
const queries = await resolveQueriesFromPacks(
cliServer,
packsToSearch,
{
kind: "table",
"tags contain all": ["modeleditor", "endpoints", modeTag(mode)],
},
additionalPackPaths,
);
if (queries.length > 1) {
throw new Error(
`Found multiple endpoints queries for ${mode}. Can't continue`,
);
}
if (queries.length === 0) {
return undefined;
}
return queries[0];
}

41
extensions/ql-vscode/src/model-editor/model-editor-view-tracker.ts Normal file
View File

@@ -0,0 +1,41 @@
import { Method } from "./method";
interface ModelEditorViewInterface {
databaseUri: string;
revealMethod(method: Method): Promise<void>;
}
export class ModelEditorViewTracker<
T extends ModelEditorViewInterface = ModelEditorViewInterface,
> {
private readonly views = new Map<string, T[]>();
constructor() {}
public registerView(view: T): void {
const databaseUri = view.databaseUri;
if (!this.views.has(databaseUri)) {
this.views.set(databaseUri, []);
}
this.views.get(databaseUri)?.push(view);
}
public unregisterView(view: T): void {
const views = this.views.get(view.databaseUri);
if (!views) {
return;
}
const index = views.indexOf(view);
if (index !== -1) {
views.splice(index, 1);
}
}
public getViews(databaseUri: string): T[] {
return this.views.get(databaseUri) ?? [];
}
}

223
extensions/ql-vscode/src/model-editor/model-editor-view.ts
View File

@@ -1,4 +1,11 @@
import { CancellationTokenSource, Uri, ViewColumn, window } from "vscode";
import {
CancellationTokenSource,
Tab,
TabInputWebview,
Uri,
ViewColumn,
window,
} from "vscode";
import {
AbstractWebview,
WebviewPanelConfig,
@@ -19,7 +26,6 @@ import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
import { runFlowModelQueries } from "./flow-model-queries";
import { promptImportGithubDatabase } from "../databases/database-fetcher";
import { App } from "../common/app";
import { showResolvableLocation } from "../databases/local-databases/locations";
import { redactableError } from "../common/errors";
import {
externalApiQueriesProgressMaxStep,
@@ -28,14 +34,19 @@ import {
import { Method, Usage } from "./method";
import { ModeledMethod } from "./modeled-method";
import { ExtensionPack } from "./shared/extension-pack";
import { showFlowGeneration, showLlmGeneration } from "../config";
import {
showFlowGeneration,
showLlmGeneration,
showMultipleModels,
} from "../config";
import { Mode } from "./shared/mode";
import { loadModeledMethods, saveModeledMethods } from "./modeled-method-fs";
import { pickExtensionPack } from "./extension-pack-picker";
import { getLanguageDisplayName } from "../common/query-language";
import { AutoModeler } from "./auto-modeler";
import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "./shared/hide-modeled-methods";
import { telemetryListener } from "../common/vscode/telemetry";
import { ModelingStore } from "./modeling-store";
import { ModelEditorViewTracker } from "./model-editor-view-tracker";
export class ModelEditorView extends AbstractWebview<
ToModelEditorMessage,
@@ -43,11 +54,10 @@ export class ModelEditorView extends AbstractWebview<
> {
private readonly autoModeler: AutoModeler;
private methods: Method[];
private hideModeledMethods: boolean;
public constructor(
protected readonly app: App,
private readonly modelingStore: ModelingStore,
private readonly viewTracker: ModelEditorViewTracker<ModelEditorView>,
private readonly databaseManager: DatabaseManager,
private readonly cliServer: CodeQLCliServer,
private readonly queryRunner: QueryRunner,
@@ -56,23 +66,14 @@ export class ModelEditorView extends AbstractWebview<
private readonly databaseItem: DatabaseItem,
private readonly extensionPack: ExtensionPack,
private mode: Mode,
private readonly updateMethodsUsagePanelState: (
methods: Method[],
databaseItem: DatabaseItem,
hideModeledMethods: boolean,
) => Promise<void>,
private readonly showMethod: (
method: Method,
usage: Usage,
) => Promise<void>,
private readonly handleViewBecameActive: (view: ModelEditorView) => void,
private readonly handleViewWasDisposed: (view: ModelEditorView) => void,
private readonly isMostRecentlyActiveView: (
view: ModelEditorView,
) => boolean,
) {
super(app);
this.modelingStore.initializeStateForDb(databaseItem);
this.registerToModelingStoreEvents();
this.viewTracker.registerView(this);
this.autoModeler = new AutoModeler(
app,
cliServer,
@@ -87,11 +88,9 @@ export class ModelEditorView extends AbstractWebview<
});
},
async (modeledMethods) => {
await this.postMessage({ t: "addModeledMethods", modeledMethods });
this.addModeledMethods(modeledMethods);
},
);
this.methods = [];
this.hideModeledMethods = INITIAL_HIDE_MODELED_METHODS_VALUE;
}
public async openView() {
@@ -100,17 +99,15 @@ export class ModelEditorView extends AbstractWebview<
panel.onDidChangeViewState(async () => {
if (panel.active) {
this.handleViewBecameActive(this);
await this.updateMethodsUsagePanelState(
this.methods,
this.databaseItem,
this.hideModeledMethods,
);
this.modelingStore.setActiveDb(this.databaseItem);
await this.markModelEditorAsActive();
} else {
await this.updateModelEditorActiveContext();
}
});
panel.onDidDispose(() => {
this.handleViewWasDisposed(this);
this.modelingStore.removeDb(this.databaseItem);
// onDidDispose is called after the tab has been closed,
// so we want to check if there are any others still open.
void this.app.commands.execute(
@@ -129,17 +126,46 @@ export class ModelEditorView extends AbstractWebview<
);
}
private async markModelEditorAsActive(): Promise<void> {
void this.app.commands.execute(
"setContext",
"codeql.modelEditorActive",
true,
);
}
private async updateModelEditorActiveContext(): Promise<void> {
await this.app.commands.execute(
"setContext",
"codeql.modelEditorActive",
this.isAModelEditorActive(),
);
}
private isAModelEditorOpen(): boolean {
return window.tabGroups.all.some((tabGroup) =>
tabGroup.tabs.some((tab) => {
const viewType: string | undefined = (tab.input as any)?.viewType;
// The viewType has a prefix, such as "mainThreadWebview-", but if the
// suffix matches that should be enough to identify the view.
return viewType && viewType.endsWith("model-editor");
}),
tabGroup.tabs.some((tab) => this.isTabModelEditorView(tab)),
);
}
private isAModelEditorActive(): boolean {
return window.tabGroups.all.some((tabGroup) =>
tabGroup.tabs.some(
(tab) => this.isTabModelEditorView(tab) && tab.isActive,
),
);
}
private isTabModelEditorView(tab: Tab): boolean {
if (!(tab.input instanceof TabInputWebview)) {
return false;
}
// The viewType has a prefix, such as "mainThreadWebview-", but if the
// suffix matches that should be enough to identify the view.
return tab.input.viewType.endsWith("model-editor");
}
protected async getPanelConfig(): Promise<WebviewPanelConfig> {
return {
viewId: "model-editor",
@@ -163,7 +189,7 @@ export class ModelEditorView extends AbstractWebview<
}
protected onPanelDispose(): void {
// Nothing to do here
this.viewTracker.unregisterView(this);
}
protected async onMessage(msg: FromModelEditorMessage): Promise<void> {
@@ -234,6 +260,12 @@ export class ModelEditorView extends AbstractWebview<
cancellable: false,
},
);
this.modelingStore.removeModifiedMethods(
this.databaseItem,
Object.keys(msg.modeledMethods),
);
void telemetryListener?.sendUIInteraction(
"model-editor-save-modeled-methods",
);
@@ -270,11 +302,11 @@ export class ModelEditorView extends AbstractWebview<
break;
case "switchMode":
this.mode = msg.mode;
this.methods = [];
this.modelingStore.setMethods(this.databaseItem, []);
await Promise.all([
this.postMessage({
t: "setMethods",
methods: this.methods,
methods: [],
}),
this.setViewState(),
withProgress((progress) => this.loadMethods(progress), {
@@ -285,16 +317,18 @@ export class ModelEditorView extends AbstractWebview<
break;
case "hideModeledMethods":
this.hideModeledMethods = msg.hideModeledMethods;
await this.updateMethodsUsagePanelState(
this.methods,
this.modelingStore.setHideModeledMethods(
this.databaseItem,
this.hideModeledMethods,
msg.hideModeledMethods,
);
void telemetryListener?.sendUIInteraction(
"model-editor-hide-modeled-methods",
);
break;
case "setModeledMethod": {
this.setModeledMethod(msg.method);
break;
}
default:
assertNever(msg);
}
@@ -312,6 +346,19 @@ export class ModelEditorView extends AbstractWebview<
]);
}
public get databaseUri(): string {
return this.databaseItem.databaseUri.toString();
}
public async revealMethod(method: Method): Promise<void> {
this.panel?.reveal();
await this.postMessage({
t: "revealMethod",
method,
});
}
private async setViewState(): Promise<void> {
const showLlmButton =
this.databaseItem.language === "java" && showLlmGeneration();
@@ -322,14 +369,14 @@ export class ModelEditorView extends AbstractWebview<
extensionPack: this.extensionPack,
showFlowGeneration: showFlowGeneration(),
showLlmButton,
showMultipleModels: showMultipleModels(),
mode: this.mode,
},
});
}
protected async handleJumpToUsage(method: Method, usage: Usage) {
await this.showMethod(method, usage);
await showResolvableLocation(usage.url, this.databaseItem, this.app.logger);
this.modelingStore.setSelectedMethod(this.databaseItem, method, usage);
}
protected async loadExistingModeledMethods(): Promise<void> {
@@ -339,10 +386,7 @@ export class ModelEditorView extends AbstractWebview<
this.cliServer,
this.app.logger,
);
await this.postMessage({
t: "loadModeledMethods",
modeledMethods,
});
this.modelingStore.setModeledMethods(this.databaseItem, modeledMethods);
} catch (e: unknown) {
void showAndLogErrorMessage(
this.app.logger,
@@ -370,19 +414,8 @@ export class ModelEditorView extends AbstractWebview<
if (!queryResult) {
return;
}
this.methods = queryResult;
await this.postMessage({
t: "setMethods",
methods: this.methods,
});
if (this.isMostRecentlyActiveView(this)) {
await this.updateMethodsUsagePanelState(
this.methods,
this.databaseItem,
this.hideModeledMethods,
);
}
this.modelingStore.setMethods(this.databaseItem, queryResult);
} catch (err) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
@@ -431,10 +464,7 @@ export class ModelEditorView extends AbstractWebview<
modeledMethodsByName[modeledMethod.signature] = modeledMethod;
}
await this.postMessage({
t: "addModeledMethods",
modeledMethods: modeledMethodsByName,
});
this.addModeledMethods(modeledMethodsByName);
},
progress,
token: tokenSource.token,
@@ -488,6 +518,8 @@ export class ModelEditorView extends AbstractWebview<
const view = new ModelEditorView(
this.app,
this.modelingStore,
this.viewTracker,
this.databaseManager,
this.cliServer,
this.queryRunner,
@@ -496,11 +528,6 @@ export class ModelEditorView extends AbstractWebview<
addedDatabase,
modelFile,
Mode.Framework,
this.updateMethodsUsagePanelState,
this.showMethod,
this.handleViewBecameActive,
this.handleViewWasDisposed,
this.isMostRecentlyActiveView,
);
await view.openView();
});
@@ -578,4 +605,58 @@ export class ModelEditorView extends AbstractWebview<
return addedDatabase;
}
private registerToModelingStoreEvents() {
this.push(
this.modelingStore.onMethodsChanged(async (event) => {
if (event.dbUri === this.databaseItem.databaseUri.toString()) {
await this.postMessage({
t: "setMethods",
methods: event.methods,
});
}
}),
);
this.push(
this.modelingStore.onModeledMethodsChanged(async (event) => {
if (event.dbUri === this.databaseItem.databaseUri.toString()) {
await this.postMessage({
t: "setModeledMethods",
methods: event.modeledMethods,
});
}
}),
);
this.push(
this.modelingStore.onModifiedMethodsChanged(async (event) => {
if (event.dbUri === this.databaseItem.databaseUri.toString()) {
await this.postMessage({
t: "setModifiedMethods",
methodSignatures: [...event.modifiedMethods],
});
}
}),
);
}
private addModeledMethods(modeledMethods: Record<string, ModeledMethod>) {
this.modelingStore.addModeledMethods(this.databaseItem, modeledMethods);
this.modelingStore.addModifiedMethods(
this.databaseItem,
new Set(Object.keys(modeledMethods)),
);
}
private setModeledMethod(method: ModeledMethod) {
const state = this.modelingStore.getStateForActiveDb();
if (!state) {
throw new Error("Attempting to set modeled method without active db");
}
this.modelingStore.updateModeledMethod(state.databaseItem, method);
this.modelingStore.addModifiedMethod(state.databaseItem, method.signature);
}
}

45
extensions/ql-vscode/src/model-editor/model-extension-file.schema.json Normal file
View File

@@ -0,0 +1,45 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/ModelExtensionFile",
"definitions": {
"ModelExtensionFile": {
"type": "object",
"properties": {
"extensions": {
"type": "array",
"items": {
"type": "object",
"properties": {
"addsTo": {
"type": "object",
"properties": {
"pack": {
"type": "string"
},
"extensible": {
"type": "string"
}
},
"required": ["pack", "extensible"]
},
"data": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/DataTuple"
}
}
}
},
"required": ["addsTo", "data"]
}
}
},
"required": ["extensions"]
},
"DataTuple": {
"type": ["boolean", "number", "string"]
}
}
}

17
extensions/ql-vscode/src/model-editor/model-extension-file.ts Normal file
View File

@@ -0,0 +1,17 @@
type ExtensibleReference = {
pack: string;
extensible: string;
};
export type DataTuple = boolean | number | string;
type DataRow = DataTuple[];
type ModelExtension = {
addsTo: ExtensibleReference;
data: DataRow[];
};
export type ModelExtensionFile = {
extensions: ModelExtension[];
};

12
extensions/ql-vscode/src/model-editor/modeled-method-fs.ts
View File

@@ -10,6 +10,11 @@ import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { load as loadYaml } from "js-yaml";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { pathsEqual } from "../common/files";
import {
convertFromLegacyModeledMethods,
convertFromLegacyModeledMethodsFiles,
convertToLegacyModeledMethods,
} from "./modeled-methods-legacy";
export async function saveModeledMethods(
extensionPack: ExtensionPack,
@@ -29,8 +34,8 @@ export async function saveModeledMethods(
const yamls = createDataExtensionYamls(
language,
methods,
modeledMethods,
existingModeledMethods,
convertFromLegacyModeledMethods(modeledMethods),
convertFromLegacyModeledMethodsFiles(existingModeledMethods),
mode,
);
@@ -68,7 +73,8 @@ async function loadModeledMethodFiles(
);
continue;
}
modeledMethodsByFile[modelFile] = modeledMethods;
modeledMethodsByFile[modelFile] =
convertToLegacyModeledMethods(modeledMethods);
}
return modeledMethodsByFile;

4
extensions/ql-vscode/src/model-editor/modeled-method.ts
View File

@@ -23,6 +23,8 @@ export interface ModeledMethod extends MethodSignature {
type: ModeledMethodType;
input: string;
output: string;
kind: string;
kind: ModeledMethodKind;
provenance: Provenance;
}
export type ModeledMethodKind = string;

33
extensions/ql-vscode/src/model-editor/modeled-methods-legacy.ts Normal file
View File

@@ -0,0 +1,33 @@
import { ModeledMethod } from "./modeled-method";
export function convertFromLegacyModeledMethods(
modeledMethods: Record<string, ModeledMethod>,
): Record<string, ModeledMethod[]> {
// Convert a single ModeledMethod to an array of ModeledMethods
return Object.fromEntries(
Object.entries(modeledMethods).map(([signature, modeledMethod]) => {
return [signature, [modeledMethod]];
}),
);
}
export function convertToLegacyModeledMethods(
modeledMethods: Record<string, ModeledMethod[]>,
): Record<string, ModeledMethod> {
// Always take the first modeled method in the array
return Object.fromEntries(
Object.entries(modeledMethods).map(([signature, modeledMethods]) => {
return [signature, modeledMethods[0]];
}),
);
}
export function convertFromLegacyModeledMethodsFiles(
modeledMethods: Record<string, Record<string, ModeledMethod>>,
): Record<string, Record<string, ModeledMethod[]>> {
return Object.fromEntries(
Object.entries(modeledMethods).map(([filename, modeledMethods]) => {
return [filename, convertFromLegacyModeledMethods(modeledMethods)];
}),
);
}

333
extensions/ql-vscode/src/model-editor/modeling-store.ts Normal file
View File

@@ -0,0 +1,333 @@
import { App } from "../common/app";
import { DisposableObject } from "../common/disposable-object";
import { AppEvent, AppEventEmitter } from "../common/events";
import { DatabaseItem } from "../databases/local-databases";
import { Method, Usage } from "./method";
import { ModeledMethod } from "./modeled-method";
import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "./shared/hide-modeled-methods";
export interface DbModelingState {
databaseItem: DatabaseItem;
methods: Method[];
hideModeledMethods: boolean;
modeledMethods: Record<string, ModeledMethod>;
modifiedMethodSignatures: Set<string>;
selectedMethod: Method | undefined;
selectedUsage: Usage | undefined;
}
interface MethodsChangedEvent {
methods: Method[];
dbUri: string;
isActiveDb: boolean;
}
interface HideModeledMethodsChangedEvent {
hideModeledMethods: boolean;
isActiveDb: boolean;
}
interface ModeledMethodsChangedEvent {
modeledMethods: Record<string, ModeledMethod>;
dbUri: string;
isActiveDb: boolean;
}
interface ModifiedMethodsChangedEvent {
modifiedMethods: Set<string>;
dbUri: string;
isActiveDb: boolean;
}
interface SelectedMethodChangedEvent {
databaseItem: DatabaseItem;
method: Method;
usage: Usage;
modeledMethod: ModeledMethod | undefined;
isModified: boolean;
}
export class ModelingStore extends DisposableObject {
public readonly onActiveDbChanged: AppEvent<void>;
public readonly onDbClosed: AppEvent<string>;
public readonly onMethodsChanged: AppEvent<MethodsChangedEvent>;
public readonly onHideModeledMethodsChanged: AppEvent<HideModeledMethodsChangedEvent>;
public readonly onModeledMethodsChanged: AppEvent<ModeledMethodsChangedEvent>;
public readonly onModifiedMethodsChanged: AppEvent<ModifiedMethodsChangedEvent>;
public readonly onSelectedMethodChanged: AppEvent<SelectedMethodChangedEvent>;
private readonly state: Map<string, DbModelingState>;
private activeDb: string | undefined;
private readonly onActiveDbChangedEventEmitter: AppEventEmitter<void>;
private readonly onDbClosedEventEmitter: AppEventEmitter<string>;
private readonly onMethodsChangedEventEmitter: AppEventEmitter<MethodsChangedEvent>;
private readonly onHideModeledMethodsChangedEventEmitter: AppEventEmitter<HideModeledMethodsChangedEvent>;
private readonly onModeledMethodsChangedEventEmitter: AppEventEmitter<ModeledMethodsChangedEvent>;
private readonly onModifiedMethodsChangedEventEmitter: AppEventEmitter<ModifiedMethodsChangedEvent>;
private readonly onSelectedMethodChangedEventEmitter: AppEventEmitter<SelectedMethodChangedEvent>;
constructor(app: App) {
super();
// State initialization
this.state = new Map<string, DbModelingState>();
// Event initialization
this.onActiveDbChangedEventEmitter = this.push(
app.createEventEmitter<void>(),
);
this.onActiveDbChanged = this.onActiveDbChangedEventEmitter.event;
this.onDbClosedEventEmitter = this.push(app.createEventEmitter<string>());
this.onDbClosed = this.onDbClosedEventEmitter.event;
this.onMethodsChangedEventEmitter = this.push(
app.createEventEmitter<MethodsChangedEvent>(),
);
this.onMethodsChanged = this.onMethodsChangedEventEmitter.event;
this.onHideModeledMethodsChangedEventEmitter = this.push(
app.createEventEmitter<HideModeledMethodsChangedEvent>(),
);
this.onHideModeledMethodsChanged =
this.onHideModeledMethodsChangedEventEmitter.event;
this.onModeledMethodsChangedEventEmitter = this.push(
app.createEventEmitter<ModeledMethodsChangedEvent>(),
);
this.onModeledMethodsChanged =
this.onModeledMethodsChangedEventEmitter.event;
this.onModifiedMethodsChangedEventEmitter = this.push(
app.createEventEmitter<ModifiedMethodsChangedEvent>(),
);
this.onModifiedMethodsChanged =
this.onModifiedMethodsChangedEventEmitter.event;
this.onSelectedMethodChangedEventEmitter = this.push(
app.createEventEmitter<SelectedMethodChangedEvent>(),
);
this.onSelectedMethodChanged =
this.onSelectedMethodChangedEventEmitter.event;
}
public initializeStateForDb(databaseItem: DatabaseItem) {
const dbUri = databaseItem.databaseUri.toString();
this.state.set(dbUri, {
databaseItem,
methods: [],
hideModeledMethods: INITIAL_HIDE_MODELED_METHODS_VALUE,
modeledMethods: {},
modifiedMethodSignatures: new Set(),
selectedMethod: undefined,
selectedUsage: undefined,
});
}
public setActiveDb(databaseItem: DatabaseItem) {
this.activeDb = databaseItem.databaseUri.toString();
this.onActiveDbChangedEventEmitter.fire();
}
public removeDb(databaseItem: DatabaseItem) {
const dbUri = databaseItem.databaseUri.toString();
if (!this.state.has(dbUri)) {
throw Error("Cannot remove a database that has not been initialized");
}
if (this.activeDb === dbUri) {
this.activeDb = undefined;
this.onActiveDbChangedEventEmitter.fire();
}
this.state.delete(dbUri);
this.onDbClosedEventEmitter.fire(dbUri);
}
public getStateForActiveDb(): DbModelingState | undefined {
if (!this.activeDb) {
return undefined;
}
return this.state.get(this.activeDb);
}
public setMethods(dbItem: DatabaseItem, methods: Method[]) {
const dbState = this.getState(dbItem);
const dbUri = dbItem.databaseUri.toString();
dbState.methods = [...methods];
this.onMethodsChangedEventEmitter.fire({
methods,
dbUri,
isActiveDb: dbUri === this.activeDb,
});
}
public setHideModeledMethods(
dbItem: DatabaseItem,
hideModeledMethods: boolean,
) {
const dbState = this.getState(dbItem);
const dbUri = dbItem.databaseUri.toString();
dbState.hideModeledMethods = hideModeledMethods;
this.onHideModeledMethodsChangedEventEmitter.fire({
hideModeledMethods,
isActiveDb: dbUri === this.activeDb,
});
}
public addModeledMethods(
dbItem: DatabaseItem,
methods: Record<string, ModeledMethod>,
) {
this.changeModeledMethods(dbItem, (state) => {
const newModeledMethods = {
...methods,
...Object.fromEntries(
Object.entries(state.modeledMethods).filter(
([_, value]) => value.type !== "none",
),
),
};
state.modeledMethods = newModeledMethods;
});
}
public setModeledMethods(
dbItem: DatabaseItem,
methods: Record<string, ModeledMethod>,
) {
this.changeModeledMethods(dbItem, (state) => {
state.modeledMethods = { ...methods };
});
}
public updateModeledMethod(dbItem: DatabaseItem, method: ModeledMethod) {
this.changeModeledMethods(dbItem, (state) => {
const newModeledMethods = { ...state.modeledMethods };
newModeledMethods[method.signature] = method;
state.modeledMethods = newModeledMethods;
});
}
public setModifiedMethods(
dbItem: DatabaseItem,
methodSignatures: Set<string>,
) {
this.changeModifiedMethods(dbItem, (state) => {
state.modifiedMethodSignatures = new Set(methodSignatures);
});
}
public addModifiedMethods(
dbItem: DatabaseItem,
methodSignatures: Iterable<string>,
) {
this.changeModifiedMethods(dbItem, (state) => {
const newModifiedMethods = new Set([
...state.modifiedMethodSignatures,
...methodSignatures,
]);
state.modifiedMethodSignatures = newModifiedMethods;
});
}
public addModifiedMethod(dbItem: DatabaseItem, methodSignature: string) {
this.addModifiedMethods(dbItem, [methodSignature]);
}
public removeModifiedMethods(
dbItem: DatabaseItem,
methodSignatures: string[],
) {
this.changeModifiedMethods(dbItem, (state) => {
const newModifiedMethods = Array.from(
state.modifiedMethodSignatures,
).filter((s) => !methodSignatures.includes(s));
state.modifiedMethodSignatures = new Set(newModifiedMethods);
});
}
public setSelectedMethod(dbItem: DatabaseItem, method: Method, usage: Usage) {
const dbState = this.getState(dbItem);
dbState.selectedMethod = method;
dbState.selectedUsage = usage;
this.onSelectedMethodChangedEventEmitter.fire({
databaseItem: dbItem,
method,
usage,
modeledMethod: dbState.modeledMethods[method.signature],
isModified: dbState.modifiedMethodSignatures.has(method.signature),
});
}
public getSelectedMethodDetails() {
const dbState = this.getStateForActiveDb();
if (!dbState) {
throw new Error("No active state found in modeling store");
}
const selectedMethod = dbState.selectedMethod;
if (!selectedMethod) {
return undefined;
}
return {
method: selectedMethod,
usage: dbState.selectedUsage,
modeledMethod: dbState.modeledMethods[selectedMethod.signature],
isModified: dbState.modifiedMethodSignatures.has(
selectedMethod.signature,
),
};
}
private getState(databaseItem: DatabaseItem): DbModelingState {
if (!this.state.has(databaseItem.databaseUri.toString())) {
throw Error(
"Cannot get state for a database that has not been initialized",
);
}
return this.state.get(databaseItem.databaseUri.toString())!;
}
private changeModifiedMethods(
dbItem: DatabaseItem,
updateState: (state: DbModelingState) => void,
) {
const state = this.getState(dbItem);
updateState(state);
this.onModifiedMethodsChangedEventEmitter.fire({
modifiedMethods: state.modifiedMethodSignatures,
dbUri: dbItem.databaseUri.toString(),
isActiveDb: dbItem.databaseUri.toString() === this.activeDb,
});
}
private changeModeledMethods(
dbItem: DatabaseItem,
updateState: (state: DbModelingState) => void,
) {
const state = this.getState(dbItem);
updateState(state);
this.onModeledMethodsChangedEventEmitter.fire({
modeledMethods: state.modeledMethods,
dbUri: dbItem.databaseUri.toString(),
isActiveDb: dbItem.databaseUri.toString() === this.activeDb,
});
}
}

9
extensions/ql-vscode/src/model-editor/predicates.ts
View File

@@ -1,16 +1,15 @@
import { ModeledMethod, ModeledMethodType, Provenance } from "./modeled-method";
import { DataTuple } from "./model-extension-file";
export type ExtensiblePredicateDefinition = {
extensiblePredicate: string;
generateMethodDefinition: (method: ModeledMethod) => Tuple[];
readModeledMethod: (row: Tuple[]) => ModeledMethod;
generateMethodDefinition: (method: ModeledMethod) => DataTuple[];
readModeledMethod: (row: DataTuple[]) => ModeledMethod;
supportedKinds?: string[];
};
type Tuple = boolean | number | string;
function readRowToMethod(row: Tuple[]): string {
function readRowToMethod(row: DataTuple[]): string {
return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
}

288
extensions/ql-vscode/src/model-editor/queries/csharp.ts
View File

@@ -2,130 +2,151 @@ import { Query } from "./query";
export const fetchExternalApisQuery: Query = {
applicationModeQuery: `/**
* @name Usage of APIs coming from external libraries
* @description A list of 3rd party APIs used in the codebase.
* @tags telemetry
* @kind problem
* @id cs/telemetry/fetch-external-apis
* @name Fetch endpoints for use in the model editor (application mode)
* @description A list of 3rd party endpoints (methods and attributes) used in the codebase. Excludes test and generated code.
* @kind table
* @id csharp/utils/modeleditor/application-mode-endpoints
* @tags modeleditor endpoints application-mode
*/
private import csharp
private import AutomodelVsCode
import csharp
import ApplicationModeEndpointsQuery
import ModelEditor
class ExternalApi extends CallableMethod {
ExternalApi() {
this.isUnboundDeclaration() and
this.fromLibrary() and
this.(Modifiable).isEffectivelyPublic()
}
}
private Call aUsage(ExternalEndpoint api) { result.getTarget().getUnboundDeclaration() = api }
private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
from
ExternalApi api, string apiName, boolean supported, Call usage, string type, string classification
from ExternalEndpoint endpoint, boolean supported, Call usage, string type, string classification
where
apiName = api.getApiName() and
supported = isSupported(api) and
usage = aUsage(api) and
type = supportedType(api) and
supported = isSupported(endpoint) and
usage = aUsage(endpoint) and
type = supportedType(endpoint) and
classification = methodClassification(usage)
select usage, apiName, supported.toString(), "supported", api.dllName(), api.dllVersion(), type,
"type", classification, "classification"
select usage, endpoint.getNamespace(), endpoint.getTypeName(), endpoint.getName(),
endpoint.getParameterTypes(), supported, endpoint.dllName(), endpoint.dllVersion(), type,
classification
`,
frameworkModeQuery: `/**
* @name Public methods
* @description A list of APIs callable by consumers. Excludes test and generated code.
* @tags telemetry
* @kind problem
* @id cs/telemetry/fetch-public-methods
* @name Fetch endpoints for use in the model editor (framework mode)
* @description A list of endpoints accessible (methods and attributes) for consumers of the library. Excludes test and generated code.
* @kind table
* @id csharp/utils/modeleditor/framework-mode-endpoints
* @tags modeleditor endpoints framework-mode
*/
private import csharp
private import dotnet
private import semmle.code.csharp.frameworks.Test
private import AutomodelVsCode
import csharp
import FrameworkModeEndpointsQuery
import ModelEditor
class PublicMethod extends CallableMethod {
PublicMethod() { this.fromSource() and not this.getFile() instanceof TestFile }
}
from PublicMethod publicMethod, string apiName, boolean supported, string type
from PublicEndpointFromSource endpoint, boolean supported, string type
where
apiName = publicMethod.getApiName() and
supported = isSupported(publicMethod) and
type = supportedType(publicMethod)
select publicMethod, apiName, supported.toString(), "supported",
publicMethod.getFile().getBaseName(), "library", type, "type", "unknown", "classification"
supported = isSupported(endpoint) and
type = supportedType(endpoint)
select endpoint, endpoint.getNamespace(), endpoint.getTypeName(), endpoint.getName(),
endpoint.getParameterTypes(), supported, endpoint.getFile().getBaseName(), type
`,
dependencies: {
"AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
private import csharp
private import dotnet
private import semmle.code.csharp.dispatch.Dispatch
private import semmle.code.csharp.dataflow.ExternalFlow
private import semmle.code.csharp.dataflow.FlowSummary
private import semmle.code.csharp.dataflow.internal.DataFlowImplCommon as DataFlowImplCommon
private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
"ApplicationModeEndpointsQuery.qll": `private import csharp
private import semmle.code.csharp.dataflow.ExternalFlow as ExternalFlow
private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
private import semmle.code.csharp.frameworks.Test
private import semmle.code.csharp.security.dataflow.flowsources.Remote
pragma[nomagic]
private predicate isTestNamespace(Namespace ns) {
ns.getFullName()
.matches([
"NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
])
}
private import ModelEditor
/**
* A test library.
* A class of effectively public callables in library code.
*/
class TestLibrary extends RefType {
TestLibrary() { isTestNamespace(this.getNamespace()) }
class ExternalEndpoint extends Endpoint {
ExternalEndpoint() { this.fromLibrary() }
/** Gets a node that is an input to a call to this API. */
private ArgumentNode getAnInput() {
result
.getCall()
.(DataFlowDispatch::NonDelegateDataFlowCall)
.getATarget(_)
.getUnboundDeclaration() = this
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(Call c, DataFlowDispatch::NonDelegateDataFlowCall dc |
dc.getDispatchCall().getCall() = c and
c.getTarget().getUnboundDeclaration() = this
|
result = DataFlowDispatch::getAnOutNode(dc, _)
)
}
override predicate hasSummary() {
Endpoint.super.hasSummary()
or
defaultAdditionalTaintStep(this.getAnInput(), _)
}
override predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or ExternalFlow::sourceNode(this.getAnOutput(), _)
}
override predicate isSink() { ExternalFlow::sinkNode(this.getAnInput(), _) }
}
`,
"FrameworkModeEndpointsQuery.qll": `private import csharp
private import semmle.code.csharp.frameworks.Test
private import ModelEditor
/**
* A class of effectively public callables from source code.
*/
class PublicEndpointFromSource extends Endpoint {
PublicEndpointFromSource() { this.fromSource() and not this.getFile() instanceof TestFile }
override predicate isSource() { this instanceof SourceCallable }
override predicate isSink() { this instanceof SinkCallable }
}`,
"ModelEditor.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
private import csharp
private import semmle.code.csharp.dataflow.FlowSummary
private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.code.csharp.frameworks.Test
/** Holds if the given callable is not worth supporting. */
private predicate isUninteresting(DotNet::Declaration c) {
private predicate isUninteresting(Callable c) {
c.getDeclaringType() instanceof TestLibrary or
c.(Constructor).isParameterless() or
c.getDeclaringType() instanceof AnonymousClass
}
/**
* An callable method from either the C# Standard Library, a 3rd party library, or from the source.
* A callable method or accessor from either the C# Standard Library, a 3rd party library, or from the source.
*/
class CallableMethod extends DotNet::Declaration {
CallableMethod() {
this.(Modifiable).isEffectivelyPublic() and
not isUninteresting(this)
class Endpoint extends Callable {
Endpoint() {
[this.(Modifiable), this.(Accessor).getDeclaration()].isEffectivelyPublic() and
not isUninteresting(this) and
this.isUnboundDeclaration()
}
/**
* Gets the unbound type, name and parameter types of this API.
*/
bindingset[this]
private string getSignature() {
result =
nestedName(this.getDeclaringType().getUnboundDeclaration()) + "#" + this.getName() + "(" +
parameterQualifiedTypeNamesToString(this) + ")"
}
/**
* Gets the namespace of this API.
* Gets the namespace of this endpoint.
*/
bindingset[this]
string getNamespace() { this.getDeclaringType().hasQualifiedName(result, _) }
/**
* Gets the namespace and signature of this API.
* Gets the unbound type name of this endpoint.
*/
bindingset[this]
string getApiName() { result = this.getNamespace() + "." + this.getSignature() }
string getTypeName() { result = nestedName(this.getDeclaringType().getUnboundDeclaration()) }
/**
* Gets the parameter types of this endpoint.
*/
bindingset[this]
string getParameterTypes() { result = "(" + parameterQualifiedTypeNamesToString(this) + ")" }
private string getDllName() { result = this.getLocation().(Assembly).getName() }
@@ -143,44 +164,17 @@ class CallableMethod extends DotNet::Declaration {
not exists(this.getDllVersion()) and result = ""
}
/** Gets a node that is an input to a call to this API. */
private ArgumentNode getAnInput() {
result
.getCall()
.(DataFlowDispatch::NonDelegateDataFlowCall)
.getATarget(_)
.getUnboundDeclaration() = this
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(
Call c, DataFlowDispatch::NonDelegateDataFlowCall dc, DataFlowImplCommon::ReturnKindExt ret
|
dc.getDispatchCall().getCall() = c and
c.getTarget().getUnboundDeclaration() = this
|
result = ret.getAnOutNode(dc)
)
}
/** Holds if this API has a supported summary. */
pragma[nomagic]
predicate hasSummary() {
this instanceof SummarizedCallable
or
defaultAdditionalTaintStep(this.getAnInput(), _)
}
predicate hasSummary() { this instanceof SummarizedCallable }
/** Holds if this API is a known source. */
pragma[nomagic]
predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
}
abstract predicate isSource();
/** Holds if this API is a known sink. */
pragma[nomagic]
predicate isSink() { sinkNode(this.getAnInput(), _) }
abstract predicate isSink();
/** Holds if this API is a known neutral. */
pragma[nomagic]
@@ -195,23 +189,20 @@ class CallableMethod extends DotNet::Declaration {
}
}
boolean isSupported(CallableMethod callableMethod) {
callableMethod.isSupported() and result = true
or
not callableMethod.isSupported() and
result = false
boolean isSupported(Endpoint endpoint) {
if endpoint.isSupported() then result = true else result = false
}
string supportedType(CallableMethod method) {
method.isSink() and result = "sink"
string supportedType(Endpoint endpoint) {
endpoint.isSink() and result = "sink"
or
method.isSource() and result = "source"
endpoint.isSource() and result = "source"
or
method.hasSummary() and result = "summary"
endpoint.hasSummary() and result = "summary"
or
method.isNeutral() and result = "neutral"
endpoint.isNeutral() and result = "neutral"
or
not method.isSupported() and result = ""
not endpoint.isSupported() and result = ""
}
string methodClassification(Call method) {
@@ -222,18 +213,51 @@ string methodClassification(Call method) {
}
/**
* Gets the nested name of the declaration.
* Gets the nested name of the type \`t\`.
*
* If the declaration is not a nested type, the result is the same as \`getName()\`.
* If the type is not a nested type, the result is the same as \`getName()\`.
* Otherwise the name of the nested type is prefixed with a \`+\` and appended to
* the name of the enclosing type, which might be a nested type as well.
*/
private string nestedName(Declaration declaration) {
not exists(declaration.getDeclaringType().getUnboundDeclaration()) and
result = declaration.getName()
private string nestedName(Type t) {
not exists(t.getDeclaringType().getUnboundDeclaration()) and
result = t.getName()
or
nestedName(declaration.getDeclaringType().getUnboundDeclaration()) + "+" + declaration.getName() =
result
nestedName(t.getDeclaringType().getUnboundDeclaration()) + "+" + t.getName() = result
}
// Temporary copy of csharp/ql/src/Telemetry/TestLibrary.qll
pragma[nomagic]
private predicate isTestNamespace(Namespace ns) {
ns.getFullName()
.matches([
"NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
])
}
/**
* A test library.
*/
class TestLibrary extends RefType {
TestLibrary() { isTestNamespace(this.getNamespace()) }
}
// Temporary copy of csharp/ql/lib/semmle/code/csharp/dataflow/ExternalFlow.qll
private import semmle.code.csharp.dataflow.internal.FlowSummaryImplSpecific
/**
* A callable where there exists a MaD sink model that applies to it.
*/
class SinkCallable extends Callable {
SinkCallable() { sinkElement(this, _, _, _) }
}
/**
* A callable where there exists a MaD source model that applies to it.
*/
class SourceCallable extends Callable {
SourceCallable() { sourceElement(this, _, _, _) }
}
`,
},

313
extensions/ql-vscode/src/model-editor/queries/java.ts
View File

@@ -2,66 +2,113 @@ import { Query } from "./query";
export const fetchExternalApisQuery: Query = {
applicationModeQuery: `/**
* @name Usage of APIs coming from external libraries
* @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
* @tags telemetry
* @kind problem
* @id java/telemetry/fetch-external-apis
* @name Fetch endpoints for use in the model editor (application mode)
* @description A list of 3rd party endpoints (methods) used in the codebase. Excludes test and generated code.
* @kind table
* @id java/utils/modeleditor/application-mode-endpoints
* @tags modeleditor endpoints application-mode
*/
import java
import AutomodelVsCode
class ExternalApi extends CallableMethod {
ExternalApi() { not this.fromSource() }
}
private Call aUsage(ExternalApi api) { result.getCallee().getSourceDeclaration() = api }
from
ExternalApi externalApi, string apiName, boolean supported, Call usage, string type,
string classification
where
apiName = externalApi.getApiName() and
supported = isSupported(externalApi) and
usage = aUsage(externalApi) and
type = supportedType(externalApi) and
classification = methodClassification(usage)
select usage, apiName, supported.toString(), "supported", externalApi.jarContainer(),
externalApi.jarVersion(), type, "type", classification, "classification"
`,
frameworkModeQuery: `/**
* @name Public methods
* @description A list of APIs callable by consumers. Excludes test and generated code.
* @tags telemetry
* @kind problem
* @id java/telemetry/fetch-public-methods
*/
import java
import AutomodelVsCode
class PublicMethodFromSource extends CallableMethod, ModelApi { }
from PublicMethodFromSource publicMethod, string apiName, boolean supported, string type
where
apiName = publicMethod.getApiName() and
supported = isSupported(publicMethod) and
type = supportedType(publicMethod)
select publicMethod, apiName, supported.toString(), "supported",
publicMethod.getCompilationUnit().getParentContainer().getBaseName(), "library", type, "type",
"unknown", "classification"
`,
dependencies: {
"AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
private import java
private import semmle.code.java.dataflow.DataFlow
private import ApplicationModeEndpointsQuery
private import ModelEditor
private Call aUsage(ExternalEndpoint endpoint) {
result.getCallee().getSourceDeclaration() = endpoint
}
from ExternalEndpoint endpoint, boolean supported, Call usage, string type, string classification
where
supported = isSupported(endpoint) and
usage = aUsage(endpoint) and
type = supportedType(endpoint) and
classification = usageClassification(usage)
select usage, endpoint.getPackageName(), endpoint.getTypeName(), endpoint.getName(),
endpoint.getParameterTypes(), supported, endpoint.jarContainer(), endpoint.jarVersion(), type,
classification
`,
frameworkModeQuery: `/**
* @name Fetch endpoints for use in the model editor (framework mode)
* @description A list of endpoints accessible (methods) for consumers of the library. Excludes test and generated code.
* @kind table
* @id java/utils/modeleditor/framework-mode-endpoints
* @tags modeleditor endpoints framework-mode
*/
private import java
private import FrameworkModeEndpointsQuery
private import ModelEditor
from PublicEndpointFromSource endpoint, boolean supported, string type
where
supported = isSupported(endpoint) and
type = supportedType(endpoint)
select endpoint, endpoint.getPackageName(), endpoint.getTypeName(), endpoint.getName(),
endpoint.getParameterTypes(), supported,
endpoint.getCompilationUnit().getParentContainer().getBaseName(), type
`,
dependencies: {
"ApplicationModeEndpointsQuery.qll": `private import java
private import semmle.code.java.dataflow.ExternalFlow
private import semmle.code.java.dataflow.FlowSources
private import semmle.code.java.dataflow.FlowSummary
private import semmle.code.java.dataflow.internal.DataFlowPrivate
private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import ModelEditor
/**
* A class of effectively public callables in library code.
*/
class ExternalEndpoint extends Endpoint {
ExternalEndpoint() { not this.fromSource() }
/** Gets a node that is an input to a call to this API. */
private DataFlow::Node getAnInput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr().(Argument).getCall() = call or
result.(ArgumentNode).getCall().asCall() = call
)
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr() = call or
result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
)
}
override predicate hasSummary() {
Endpoint.super.hasSummary()
or
TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
}
override predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
}
override predicate isSink() { sinkNode(this.getAnInput(), _) }
}
`,
"FrameworkModeEndpointsQuery.qll": `private import java
private import semmle.code.java.dataflow.internal.DataFlowPrivate
private import semmle.code.java.dataflow.internal.FlowSummaryImplSpecific
private import semmle.code.java.dataflow.internal.ModelExclusions
private import ModelEditor
/**
* A class of effectively public callables from source code.
*/
class PublicEndpointFromSource extends Endpoint, ModelApi {
override predicate isSource() { sourceElement(this, _, _, _) }
override predicate isSink() { sinkElement(this, _, _, _) }
}
`,
"ModelEditor.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
private import java
private import semmle.code.java.dataflow.ExternalFlow
private import semmle.code.java.dataflow.FlowSummary
private import semmle.code.java.dataflow.TaintTracking
private import semmle.code.java.dataflow.internal.ModelExclusions
@@ -75,17 +122,23 @@ private predicate isUninteresting(Callable c) {
/**
* A callable method from either the Standard Library, a 3rd party library or from the source.
*/
class CallableMethod extends Callable {
CallableMethod() { not isUninteresting(this) }
class Endpoint extends Callable {
Endpoint() { not isUninteresting(this) }
/**
* Gets information about the external API in the form expected by the MaD modeling framework.
* Gets the package name of this endpoint.
*/
string getApiName() {
result =
this.getDeclaringType().getPackage() + "." + this.getDeclaringType().nestedName() + "#" +
this.getName() + paramsString(this)
}
string getPackageName() { result = this.getDeclaringType().getPackage().getName() }
/**
* Gets the type name of this endpoint.
*/
string getTypeName() { result = this.getDeclaringType().nestedName() }
/**
* Gets the parameter types of this endpoint.
*/
string getParameterTypes() { result = paramsString(this) }
private string getJarName() {
result = this.getCompilationUnit().getParentContainer*().(JarFile).getBaseName()
@@ -113,43 +166,23 @@ class CallableMethod extends Callable {
not exists(this.getJarVersion()) and result = ""
}
/** Gets a node that is an input to a call to this API. */
private DataFlow::Node getAnInput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr().(Argument).getCall() = call or
result.(ArgumentNode).getCall().asCall() = call
)
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr() = call or
result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
)
}
/** Holds if this API has a supported summary. */
pragma[nomagic]
predicate hasSummary() {
this = any(SummarizedCallable sc).asCallable() or
TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
}
predicate hasSummary() { this = any(SummarizedCallable sc).asCallable() }
/** Holds if this API is a known source. */
pragma[nomagic]
predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
}
abstract predicate isSource();
/** Holds if this API is a known sink. */
pragma[nomagic]
predicate isSink() { sinkNode(this.getAnInput(), _) }
abstract predicate isSink();
/** Holds if this API is a known neutral. */
pragma[nomagic]
predicate isNeutral() {
exists(string namespace, string type, string name, string signature, string kind, string provenance |
neutralModel(namespace, type, name, signature, kind, provenance) and
exists(string namespace, string type, string name, string signature |
neutralModel(namespace, type, name, signature, _, _) and
this = interpretElement(namespace, type, false, name, signature, "")
)
}
@@ -163,108 +196,38 @@ class CallableMethod extends Callable {
}
}
boolean isSupported(CallableMethod method) {
method.isSupported() and result = true
boolean isSupported(Endpoint endpoint) {
endpoint.isSupported() and result = true
or
not method.isSupported() and result = false
not endpoint.isSupported() and result = false
}
string supportedType(CallableMethod method) {
method.isSink() and result = "sink"
string supportedType(Endpoint endpoint) {
endpoint.isSink() and result = "sink"
or
method.isSource() and result = "source"
endpoint.isSource() and result = "source"
or
method.hasSummary() and result = "summary"
endpoint.hasSummary() and result = "summary"
or
method.isNeutral() and result = "neutral"
endpoint.isNeutral() and result = "neutral"
or
not method.isSupported() and result = ""
not endpoint.isSupported() and result = ""
}
string methodClassification(Call method) {
isInTestFile(method.getLocation().getFile()) and result = "test"
string usageClassification(Call usage) {
isInTestFile(usage.getLocation().getFile()) and result = "test"
or
method.getFile() instanceof GeneratedFile and result = "generated"
usage.getFile() instanceof GeneratedFile and result = "generated"
or
not isInTestFile(method.getLocation().getFile()) and
not method.getFile() instanceof GeneratedFile and
not isInTestFile(usage.getLocation().getFile()) and
not usage.getFile() instanceof GeneratedFile and
result = "source"
}
// The below is a copy of https://github.com/github/codeql/blob/249f9f863db1e94e3c46ca85b49fb0ec32f8ca92/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll
// to avoid the use of internal modules.
/** Holds if the given package \`p\` is a test package. */
pragma[nomagic]
private predicate isTestPackage(Package p) {
p.getName()
.matches([
"org.junit%", "junit.%", "org.mockito%", "org.assertj%",
"com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
"org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
"org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
"org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
"org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
"org.testng%"
])
}
/**
* A test library.
*/
class TestLibrary extends RefType {
TestLibrary() { isTestPackage(this.getPackage()) }
}
/** Holds if the given file is a test file. */
private predicate isInTestFile(File file) {
// Temporarily copied from java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll
predicate isInTestFile(File file) {
file.getAbsolutePath().matches(["%/test/%", "%/guava-tests/%", "%/guava-testlib/%"]) and
not file.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
}
/** Holds if the given compilation unit's package is a JDK internal. */
private predicate isJdkInternal(CompilationUnit cu) {
cu.getPackage().getName().matches("org.graalvm%") or
cu.getPackage().getName().matches("com.sun%") or
cu.getPackage().getName().matches("sun%") or
cu.getPackage().getName().matches("jdk%") or
cu.getPackage().getName().matches("java2d%") or
cu.getPackage().getName().matches("build.tools%") or
cu.getPackage().getName().matches("propertiesparser%") or
cu.getPackage().getName().matches("org.jcp%") or
cu.getPackage().getName().matches("org.w3c%") or
cu.getPackage().getName().matches("org.ietf.jgss%") or
cu.getPackage().getName().matches("org.xml.sax%") or
cu.getPackage().getName().matches("com.oracle%") or
cu.getPackage().getName().matches("org.omg%") or
cu.getPackage().getName().matches("org.relaxng%") or
cu.getPackage().getName() = "compileproperties" or
cu.getPackage().getName() = "transparentruler" or
cu.getPackage().getName() = "genstubs" or
cu.getPackage().getName() = "netscape.javascript" or
cu.getPackage().getName() = ""
}
/** Holds if the given callable is not worth modeling. */
predicate isUninterestingForModels(Callable c) {
isInTestFile(c.getCompilationUnit().getFile()) or
isJdkInternal(c.getCompilationUnit()) or
c instanceof MainMethod or
c instanceof StaticInitializer or
exists(FunctionalExpr funcExpr | c = funcExpr.asMethod()) or
c.getDeclaringType() instanceof TestLibrary or
c.(Constructor).isParameterless()
}
/**
* A class that represents all callables for which we might be
* interested in having a MaD model.
*/
class ModelApi extends SrcCallable {
ModelApi() {
this.fromSource() and
this.isEffectivelyPublic() and
not isUninterestingForModels(this)
}
not file.getAbsolutePath().matches(["%/ql/test/%", "%/ql/automodel/test/%"]) // allows our test cases to work
}
`,
},

49
extensions/ql-vscode/src/model-editor/queries/query.ts
View File

@@ -1,18 +1,21 @@
import { Call, CallClassification } from "../method";
import { ModeledMethodType } from "../modeled-method";
export type Query = {
/**
* The application query.
*
* It should select all usages of external APIs, and return the following result pattern:
* - usage: the usage of the external API. This is an entity.
* - apiName: the name of the external API. This is a string.
* - supported: whether the external API is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - "supported": a string literal. This is required to make the query a valid problem query.
* - packageName: the package name of the external API. This is a string.
* - typeName: the type name of the external API. This is a string.
* - methodName: the method name of the external API. This is a string.
* - methodParameters: the parameters of the external API. This is a string.
* - supported: whether the external API is modeled. This is a boolean.
* - libraryName: the name of the library that contains the external API. This is a string and usually the basename of a file.
* - libraryVersion: the version of the library that contains the external API. This is a string and can be empty if the version cannot be determined.
* - type: the modeled kind of the method, either "sink", "source", "summary", or "neutral"
* - "type": a string literal. This is required to make the query a valid problem query.
* - classification: the classification of the use of the method, either "source", "test", "generated", or "unknown"
* - "classification: a string literal. This is required to make the query a valid problem query.
*/
applicationModeQuery: string;
/**
@@ -21,18 +24,40 @@ export type Query = {
* It should select all methods that are callable by applications, which is usually all public methods (and constructors).
* The result pattern should be as follows:
* - method: the method that is callable by applications. This is an entity.
* - apiName: the name of the external API. This is a string.
* - packageName: the package name of the method. This is a string.
* - typeName: the type name of the method. This is a string.
* - methodName: the method name of the method. This is a string.
* - methodParameters: the parameters of the method. This is a string.
* - supported: whether this method is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - "supported": a string literal. This is required to make the query a valid problem query.
* - libraryName: an arbitrary string. This is required to make it match the structure of the application query.
* - libraryVersion: an arbitrary string. This is required to make it match the structure of the application query.
* - libraryName: the name of the file or library that contains the method. This is a string and usually the basename of a file.
* - type: the modeled kind of the method, either "sink", "source", "summary", or "neutral"
* - "type": a string literal. This is required to make the query a valid problem query.
* - "unknown": a string literal. This is required to make it match the structure of the application query.
* - "classification: a string literal. This is required to make the query a valid problem query.
*/
frameworkModeQuery: string;
dependencies?: {
[filename: string]: string;
};
};
export type ApplicationModeTuple = [
Call,
string,
string,
string,
string,
boolean,
string,
string,
ModeledMethodType,
CallClassification,
];
export type FrameworkModeTuple = [
Call,
string,
string,
string,
string,
boolean,
string,
ModeledMethodType,
];

17
extensions/ql-vscode/src/model-editor/shared/modeling-status.ts Normal file
View File

@@ -0,0 +1,17 @@
import { ModeledMethod } from "../modeled-method";
export type ModelingStatus = "unmodeled" | "unsaved" | "saved";
export function getModelingStatus(
modeledMethod: ModeledMethod | undefined,
methodIsUnsaved: boolean,
): ModelingStatus {
if (modeledMethod) {
if (methodIsUnsaved) {
return "unsaved";
} else if (modeledMethod.type !== "none") {
return "saved";
}
}
return "unmodeled";
}

1
extensions/ql-vscode/src/model-editor/shared/view-state.ts
View File

@@ -5,5 +5,6 @@ export interface ModelEditorViewState {
extensionPack: ExtensionPack;
showFlowGeneration: boolean;
showLlmButton: boolean;
showMultipleModels: boolean;
mode: Mode;
}

75
extensions/ql-vscode/src/model-editor/yaml.ts
View File

@@ -7,12 +7,13 @@ import {
extensiblePredicateDefinitions,
} from "./predicates";
import * as dataSchemaJson from "./data-schema.json";
import * as modelExtensionFileSchema from "./model-extension-file.schema.json";
import { Mode } from "./shared/mode";
import { assertNever } from "../common/helpers-pure";
import { ModelExtensionFile } from "./model-extension-file";
const ajv = new Ajv({ allErrors: true });
const dataSchemaValidate = ajv.compile(dataSchemaJson);
const ajv = new Ajv({ allErrors: true, allowUnionTypes: true });
const modelExtensionFileSchemaValidate = ajv.compile(modelExtensionFileSchema);
function createDataProperty(
methods: ModeledMethod[],
@@ -70,8 +71,8 @@ ${extensions.join("\n")}`;
export function createDataExtensionYamls(
language: string,
methods: Method[],
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
newModeledMethods: Record<string, ModeledMethod[]>,
existingModeledMethods: Record<string, Record<string, ModeledMethod[]>>,
mode: Mode,
) {
switch (mode) {
@@ -97,11 +98,11 @@ export function createDataExtensionYamls(
function createDataExtensionYamlsByGrouping(
language: string,
methods: Method[],
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
newModeledMethods: Record<string, ModeledMethod[]>,
existingModeledMethods: Record<string, Record<string, ModeledMethod[]>>,
createFilename: (method: Method) => string,
): Record<string, string> {
const methodsByFilename: Record<string, Record<string, ModeledMethod>> = {};
const methodsByFilename: Record<string, Record<string, ModeledMethod[]>> = {};
// We only want to generate a yaml file when it's a known external API usage
// and there are new modeled methods for it. This avoids us overwriting other
@@ -113,10 +114,12 @@ function createDataExtensionYamlsByGrouping(
}
// First populate methodsByFilename with any existing modeled methods.
for (const [filename, methods] of Object.entries(existingModeledMethods)) {
for (const [filename, methodsBySignature] of Object.entries(
existingModeledMethods,
)) {
if (filename in methodsByFilename) {
for (const [signature, method] of Object.entries(methods)) {
methodsByFilename[filename][signature] = method;
for (const [signature, methods] of Object.entries(methodsBySignature)) {
methodsByFilename[filename][signature] = methods;
}
}
}
@@ -124,10 +127,12 @@ function createDataExtensionYamlsByGrouping(
// Add the new modeled methods, potentially overwriting existing modeled methods
// but not removing existing modeled methods that are not in the new set.
for (const method of methods) {
const newMethod = newModeledMethods[method.signature];
if (newMethod) {
const newMethods = newModeledMethods[method.signature];
if (newMethods) {
const filename = createFilename(method);
methodsByFilename[filename][newMethod.signature] = newMethod;
// Override any existing modeled methods with the new ones.
methodsByFilename[filename][method.signature] = newMethods;
}
}
@@ -136,7 +141,7 @@ function createDataExtensionYamlsByGrouping(
for (const [filename, methods] of Object.entries(methodsByFilename)) {
result[filename] = createDataExtensionYaml(
language,
Object.values(methods),
Object.values(methods).flatMap((methods) => methods),
);
}
@@ -146,8 +151,8 @@ function createDataExtensionYamlsByGrouping(
export function createDataExtensionYamlsForApplicationMode(
language: string,
methods: Method[],
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
newModeledMethods: Record<string, ModeledMethod[]>,
existingModeledMethods: Record<string, Record<string, ModeledMethod[]>>,
): Record<string, string> {
return createDataExtensionYamlsByGrouping(
language,
@@ -161,8 +166,8 @@ export function createDataExtensionYamlsForApplicationMode(
export function createDataExtensionYamlsForFrameworkMode(
language: string,
methods: Method[],
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
newModeledMethods: Record<string, ModeledMethod[]>,
existingModeledMethods: Record<string, Record<string, ModeledMethod[]>>,
): Record<string, string> {
return createDataExtensionYamlsByGrouping(
language,
@@ -211,25 +216,30 @@ export function createFilenameForPackage(
return `${prefix}${packageName}${suffix}.yml`;
}
export function loadDataExtensionYaml(
data: any,
): Record<string, ModeledMethod> | undefined {
dataSchemaValidate(data);
function validateModelExtensionFile(data: unknown): data is ModelExtensionFile {
modelExtensionFileSchemaValidate(data);
if (dataSchemaValidate.errors) {
if (modelExtensionFileSchemaValidate.errors) {
throw new Error(
`Invalid data extension YAML: ${dataSchemaValidate.errors
`Invalid data extension YAML: ${modelExtensionFileSchemaValidate.errors
.map((error) => `${error.instancePath} ${error.message}`)
.join(", ")}`,
);
}
const extensions = data.extensions;
if (!Array.isArray(extensions)) {
return true;
}
export function loadDataExtensionYaml(
data: unknown,
): Record<string, ModeledMethod[]> | undefined {
if (!validateModelExtensionFile(data)) {
return undefined;
}
const modeledMethods: Record<string, ModeledMethod> = {};
const extensions = data.extensions;
const modeledMethods: Record<string, ModeledMethod[]> = {};
for (const extension of extensions) {
const addsTo = extension.addsTo;
@@ -244,11 +254,16 @@ export function loadDataExtensionYaml(
}
for (const row of data) {
const modeledMethod = definition.readModeledMethod(row);
const modeledMethod: ModeledMethod = definition.readModeledMethod(row);
if (!modeledMethod) {
continue;
}
modeledMethods[modeledMethod.signature] = modeledMethod;
if (!(modeledMethod.signature in modeledMethods)) {
modeledMethods[modeledMethod.signature] = [];
}
modeledMethods[modeledMethod.signature].push(modeledMethod);
}
}

2
extensions/ql-vscode/src/packages/commands/CommandManager.ts
View File

@@ -23,8 +23,6 @@ export class CommandManager<
CommandName extends keyof Commands & string = keyof Commands & string,
> implements Disposable
{
// TODO: should this be a map?
// TODO: handle multiple command names
private commands: Disposable[] = [];
constructor(

16
extensions/ql-vscode/src/packaging/qlpack-file.ts Normal file
View File

@@ -0,0 +1,16 @@
import { SuiteInstruction } from "./suite-instruction";
/**
* The qlpack pack file, either in qlpack.yml or in codeql-pack.yml.
*/
export interface QlPackFile {
name: string;
version: string;
dependencies?: Record<string, string>;
extensionTargets?: Record<string, string>;
dbscheme?: string;
library?: boolean;
defaultSuite?: SuiteInstruction[];
defaultSuiteFile?: string;
dataExtensions?: string[] | string;
}

8
extensions/ql-vscode/src/packaging/qlpack-lock-file.ts Normal file
View File

@@ -0,0 +1,8 @@
/**
* The qlpack lock file, either in qlpack.lock.yml or in codeql-pack.lock.yml.
*/
export interface QlPackLockFile {
lockVersion: string;
dependencies?: Record<string, string>;
compiled?: boolean;
}

12
extensions/ql-vscode/src/packaging/suite-instruction.ts Normal file
View File

@@ -0,0 +1,12 @@
/**
* A single entry in a .qls file.
*/
export interface SuiteInstruction {
qlpack?: string;
query?: string;
queries?: string;
include?: Record<string, string[]>;
exclude?: Record<string, string[]>;
description?: string;
from?: string;
}

13
extensions/ql-vscode/src/queries-panel/queries-module.ts
View File

@@ -6,6 +6,7 @@ import { DisposableObject } from "../common/disposable-object";
import { QueriesPanel } from "./queries-panel";
import { QueryDiscovery } from "./query-discovery";
import { QueryPackDiscovery } from "./query-pack-discovery";
import { LanguageContextStore } from "../language-context-store";
export class QueriesModule extends DisposableObject {
private queriesPanel: QueriesPanel | undefined;
@@ -16,16 +17,21 @@ export class QueriesModule extends DisposableObject {
public static initialize(
app: App,
languageContext: LanguageContextStore,
cliServer: CodeQLCliServer,
): QueriesModule {
const queriesModule = new QueriesModule(app);
app.subscriptions.push(queriesModule);
queriesModule.initialize(app, cliServer);
queriesModule.initialize(app, languageContext, cliServer);
return queriesModule;
}
private initialize(app: App, cliServer: CodeQLCliServer): void {
private initialize(
app: App,
langauageContext: LanguageContextStore,
cliServer: CodeQLCliServer,
): void {
// Currently, we only want to expose the new panel when we are in canary mode
// and the user has enabled the "Show queries panel" flag.
if (!isCanary() || !showQueriesPanel()) {
@@ -38,8 +44,9 @@ export class QueriesModule extends DisposableObject {
void queryPackDiscovery.initialRefresh();
const queryDiscovery = new QueryDiscovery(
app.environment,
app,
queryPackDiscovery,
langauageContext,
);
this.push(queryDiscovery);
void queryDiscovery.initialRefresh();

40
extensions/ql-vscode/src/queries-panel/query-discovery.ts
View File

@@ -1,6 +1,6 @@
import { dirname, basename, normalize, relative } from "path";
import { Event } from "vscode";
import { EnvironmentContext } from "../common/app";
import { App } from "../common/app";
import {
FileTreeDirectory,
FileTreeLeaf,
@@ -11,6 +11,8 @@ import { FilePathDiscovery } from "../common/vscode/file-path-discovery";
import { containsPath } from "../common/files";
import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
import { QueryLanguage } from "../common/query-language";
import { LanguageContextStore } from "../language-context-store";
import { AppEvent, AppEventEmitter } from "../common/events";
const QUERY_FILE_EXTENSION = ".ql";
@@ -31,24 +33,36 @@ export class QueryDiscovery
extends FilePathDiscovery<Query>
implements QueryDiscoverer
{
public readonly onDidChangeQueries: AppEvent<void>;
private readonly onDidChangeQueriesEmitter: AppEventEmitter<void>;
constructor(
private readonly env: EnvironmentContext,
private readonly app: App,
private readonly queryPackDiscovery: QueryPackDiscoverer,
private readonly languageContext: LanguageContextStore,
) {
super("Query Discovery", `**/*${QUERY_FILE_EXTENSION}`);
// Set up event emitters
this.onDidChangeQueriesEmitter = this.push(app.createEventEmitter<void>());
this.onDidChangeQueries = this.onDidChangeQueriesEmitter.event;
// Handlers
this.push(
this.queryPackDiscovery.onDidChangeQueryPacks(
this.recomputeAllData.bind(this),
),
);
}
/**
* Event that fires when the set of queries in the workspace changes.
*/
public get onDidChangeQueries(): Event<void> {
return this.onDidChangePathData;
this.push(
this.onDidChangePathData(() => {
this.onDidChangeQueriesEmitter.fire();
}),
);
this.push(
this.languageContext.onLanguageContextChanged(() => {
this.onDidChangeQueriesEmitter.fire();
}),
);
}
/**
@@ -64,8 +78,10 @@ export class QueryDiscovery
const roots = [];
for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
const queriesInRoot = pathData.filter((query) =>
containsPath(workspaceFolder.uri.fsPath, query.path),
const queriesInRoot = pathData.filter(
(query) =>
containsPath(workspaceFolder.uri.fsPath, query.path) &&
this.languageContext.shouldInclude(query.language),
);
if (queriesInRoot.length === 0) {
continue;
@@ -73,7 +89,7 @@ export class QueryDiscovery
const root = new FileTreeDirectory<string>(
workspaceFolder.uri.fsPath,
workspaceFolder.name,
this.env,
this.app.environment,
);
for (const query of queriesInRoot) {
const dirName = dirname(normalize(relative(root.path, query.path)));

9
extensions/ql-vscode/src/query-history/history-item-label-provider.ts
View File

@@ -4,6 +4,7 @@ import { QueryHistoryConfig } from "../config";
import { LocalQueryInfo } from "../query-results";
import {
buildRepoLabel,
getLanguage,
getRawQueryName,
QueryHistoryInfo,
} from "./query-history-info";
@@ -19,6 +20,7 @@ interface InterpolateReplacements {
r: string; // Result count/Empty
s: string; // Status
f: string; // Query file name
l: string; // Query language
"%": "%"; // Percent sign
}
@@ -84,6 +86,7 @@ export class HistoryItemLabelProvider {
r: `(${resultCount} results)`,
s: statusString,
f: item.getQueryFileName(),
l: this.getLanguageLabel(item),
"%": "%",
};
}
@@ -103,7 +106,13 @@ export class HistoryItemLabelProvider {
r: resultCount,
s: humanizeQueryStatus(item.status),
f: basename(item.variantAnalysis.query.filePath),
l: this.getLanguageLabel(item),
"%": "%",
};
}
private getLanguageLabel(item: QueryHistoryInfo): string {
const language = getLanguage(item);
return language === undefined ? "unknown" : `${language}`;
}
}

84
extensions/ql-vscode/src/query-history/history-tree-data-provider.ts
View File

@@ -10,9 +10,10 @@ import {
} from "vscode";
import { DisposableObject } from "../common/disposable-object";
import { assertNever } from "../common/helpers-pure";
import { QueryHistoryInfo } from "./query-history-info";
import { getLanguage, QueryHistoryInfo } from "./query-history-info";
import { QueryStatus } from "./query-status";
import { HistoryItemLabelProvider } from "./history-item-label-provider";
import { LanguageContextStore } from "../language-context-store";
export enum SortOrder {
NameAsc = "NameAsc",
@@ -50,7 +51,10 @@ export class HistoryTreeDataProvider
private current: QueryHistoryInfo | undefined;
constructor(private readonly labelProvider: HistoryItemLabelProvider) {
constructor(
private readonly labelProvider: HistoryItemLabelProvider,
private readonly languageContext: LanguageContextStore,
) {
super();
}
@@ -127,51 +131,55 @@ export class HistoryTreeDataProvider
getChildren(element?: QueryHistoryInfo): ProviderResult<QueryHistoryInfo[]> {
return element
? []
: this.history.sort((h1, h2) => {
const h1Label = this.labelProvider.getLabel(h1).toLowerCase();
const h2Label = this.labelProvider.getLabel(h2).toLowerCase();
: this.history
.filter((h) => {
return this.languageContext.shouldInclude(getLanguage(h));
})
.sort((h1, h2) => {
const h1Label = this.labelProvider.getLabel(h1).toLowerCase();
const h2Label = this.labelProvider.getLabel(h2).toLowerCase();
const h1Date = this.getItemDate(h1);
const h1Date = this.getItemDate(h1);
const h2Date = this.getItemDate(h2);
const h2Date = this.getItemDate(h2);
const resultCount1 =
h1.t === "local"
? h1.completedQuery?.resultCount ?? -1
: h1.resultCount ?? -1;
const resultCount2 =
h2.t === "local"
? h2.completedQuery?.resultCount ?? -1
: h2.resultCount ?? -1;
const resultCount1 =
h1.t === "local"
? h1.completedQuery?.resultCount ?? -1
: h1.resultCount ?? -1;
const resultCount2 =
h2.t === "local"
? h2.completedQuery?.resultCount ?? -1
: h2.resultCount ?? -1;
switch (this.sortOrder) {
case SortOrder.NameAsc:
return h1Label.localeCompare(h2Label, env.language);
switch (this.sortOrder) {
case SortOrder.NameAsc:
return h1Label.localeCompare(h2Label, env.language);
case SortOrder.NameDesc:
return h2Label.localeCompare(h1Label, env.language);
case SortOrder.NameDesc:
return h2Label.localeCompare(h1Label, env.language);
case SortOrder.DateAsc:
return h1Date - h2Date;
case SortOrder.DateAsc:
return h1Date - h2Date;
case SortOrder.DateDesc:
return h2Date - h1Date;
case SortOrder.DateDesc:
return h2Date - h1Date;
case SortOrder.CountAsc:
// If the result counts are equal, sort by name.
return resultCount1 - resultCount2 === 0
? h1Label.localeCompare(h2Label, env.language)
: resultCount1 - resultCount2;
case SortOrder.CountAsc:
// If the result counts are equal, sort by name.
return resultCount1 - resultCount2 === 0
? h1Label.localeCompare(h2Label, env.language)
: resultCount1 - resultCount2;
case SortOrder.CountDesc:
// If the result counts are equal, sort by name.
return resultCount2 - resultCount1 === 0
? h2Label.localeCompare(h1Label, env.language)
: resultCount2 - resultCount1;
default:
assertNever(this.sortOrder);
}
});
case SortOrder.CountDesc:
// If the result counts are equal, sort by name.
return resultCount2 - resultCount1 === 0
? h2Label.localeCompare(h1Label, env.language)
: resultCount2 - resultCount1;
default:
assertNever(this.sortOrder);
}
});
}
getParent(_element: QueryHistoryInfo): ProviderResult<QueryHistoryInfo> {

12
extensions/ql-vscode/src/query-history/query-history-info.ts
View File

@@ -6,6 +6,7 @@ import {
hasRepoScanCompleted,
getActionsWorkflowRunUrl as getVariantAnalysisActionsWorkflowRunUrl,
} from "../variant-analysis/shared/variant-analysis";
import { QueryLanguage } from "../common/query-language";
export type QueryHistoryInfo = LocalQueryInfo | VariantAnalysisHistoryItem;
@@ -49,6 +50,17 @@ export function getQueryText(item: QueryHistoryInfo): string {
}
}
export function getLanguage(item: QueryHistoryInfo): QueryLanguage | undefined {
switch (item.t) {
case "local":
return item.initialInfo.databaseInfo.language;
case "variant-analysis":
return item.variantAnalysis.query.language;
default:
assertNever(item);
}
}
export function buildRepoLabel(item: VariantAnalysisHistoryItem): string {
const totalScannedRepositoryCount =
item.variantAnalysis.scannedRepos?.length ?? 0;

10
extensions/ql-vscode/src/query-history/query-history-manager.ts
View File

@@ -62,6 +62,7 @@ import {
showAndLogInformationMessage,
showAndLogWarningMessage,
} from "../common/logging";
import { LanguageContextStore } from "../language-context-store";
/**
* query-history-manager.ts
@@ -141,6 +142,7 @@ export class QueryHistoryManager extends DisposableObject {
ctx: ExtensionContext,
private readonly queryHistoryConfigListener: QueryHistoryConfig,
private readonly labelProvider: HistoryItemLabelProvider,
private readonly languageContext: LanguageContextStore,
private readonly doCompareCallback: (
from: CompletedLocalQueryInfo,
to: CompletedLocalQueryInfo,
@@ -158,7 +160,7 @@ export class QueryHistoryManager extends DisposableObject {
);
this.treeDataProvider = this.push(
new HistoryTreeDataProvider(this.labelProvider),
new HistoryTreeDataProvider(this.labelProvider, this.languageContext),
);
this.treeView = this.push(
window.createTreeView("codeQLQueryHistory", {
@@ -230,6 +232,12 @@ export class QueryHistoryManager extends DisposableObject {
this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
this.registerToVariantAnalysisEvents();
this.push(
this.languageContext.onLanguageContextChanged(async () => {
this.treeDataProvider.refresh();
}),
);
}
public getCommands(): QueryHistoryCommands {

28
extensions/ql-vscode/src/query-history/store/query-history-domain-mapper.ts
View File

@@ -1,8 +1,9 @@
import { assertNever } from "../../common/helpers-pure";
import { QueryHistoryInfo } from "../query-history-info";
import { mapLocalQueryInfoToDto } from "./query-history-local-query-domain-mapper";
import { QueryHistoryItemDto } from "./query-history-dto";
import { QueryHistoryItemDto, QueryLanguageDto } from "./query-history-dto";
import { mapQueryHistoryVariantAnalysisToDto } from "./query-history-variant-analysis-domain-mapper";
import { QueryLanguage } from "../../common/query-language";
export function mapQueryHistoryToDto(
queries: QueryHistoryInfo[],
@@ -17,3 +18,28 @@ export function mapQueryHistoryToDto(
}
});
}
export function mapQueryLanguageToDto(
language: QueryLanguage,
): QueryLanguageDto {
switch (language) {
case QueryLanguage.CSharp:
return QueryLanguageDto.CSharp;
case QueryLanguage.Cpp:
return QueryLanguageDto.Cpp;
case QueryLanguage.Go:
return QueryLanguageDto.Go;
case QueryLanguage.Java:
return QueryLanguageDto.Java;
case QueryLanguage.Javascript:
return QueryLanguageDto.Javascript;
case QueryLanguage.Python:
return QueryLanguageDto.Python;
case QueryLanguage.Ruby:
return QueryLanguageDto.Ruby;
case QueryLanguage.Swift:
return QueryLanguageDto.Swift;
default:
assertNever(language);
}
}

29
extensions/ql-vscode/src/query-history/store/query-history-dto-mapper.ts
View File

@@ -1,7 +1,9 @@
import { QueryHistoryInfo } from "../query-history-info";
import { QueryHistoryItemDto } from "./query-history-dto";
import { QueryHistoryItemDto, QueryLanguageDto } from "./query-history-dto";
import { mapQueryHistoryVariantAnalysisToDomainModel } from "./query-history-variant-analysis-dto-mapper";
import { mapLocalQueryItemToDomainModel } from "./query-history-local-query-dto-mapper";
import { QueryLanguage } from "../../common/query-language";
import { assertNever } from "../../common/helpers-pure";
export function mapQueryHistoryToDomainModel(
queries: QueryHistoryItemDto[],
@@ -20,3 +22,28 @@ export function mapQueryHistoryToDomainModel(
);
});
}
export function mapQueryLanguageToDomainModel(
language: QueryLanguageDto,
): QueryLanguage {
switch (language) {
case QueryLanguageDto.CSharp:
return QueryLanguage.CSharp;
case QueryLanguageDto.Cpp:
return QueryLanguage.Cpp;
case QueryLanguageDto.Go:
return QueryLanguage.Go;
case QueryLanguageDto.Java:
return QueryLanguage.Java;
case QueryLanguageDto.Javascript:
return QueryLanguage.Javascript;
case QueryLanguageDto.Python:
return QueryLanguage.Python;
case QueryLanguageDto.Ruby:
return QueryLanguage.Ruby;
case QueryLanguageDto.Swift:
return QueryLanguage.Swift;
default:
assertNever(language);
}
}

11
extensions/ql-vscode/src/query-history/store/query-history-dto.ts
View File

@@ -12,3 +12,14 @@ export interface QueryHistoryDto {
export type QueryHistoryItemDto =
| QueryHistoryLocalQueryDto
| QueryHistoryVariantAnalysisDto;
export enum QueryLanguageDto {
CSharp = "csharp",
Cpp = "cpp",
Go = "go",
Java = "java",
Javascript = "javascript",
Python = "python",
Ruby = "ruby",
Swift = "swift",
}

5
extensions/ql-vscode/src/query-history/store/query-history-local-query-domain-mapper.ts
View File

@@ -17,6 +17,7 @@ import {
SortDirection,
SortedResultSetInfo,
} from "../../common/interface-types";
import { mapQueryLanguageToDto } from "./query-history-domain-mapper";
export function mapLocalQueryInfoToDto(
query: LocalQueryInfo,
@@ -101,6 +102,10 @@ function mapInitialQueryInfoToDto(
databaseInfo: {
databaseUri: localQueryInitialInfo.databaseInfo.databaseUri,
name: localQueryInitialInfo.databaseInfo.name,
language:
localQueryInitialInfo.databaseInfo.language === undefined
? undefined
: mapQueryLanguageToDto(localQueryInitialInfo.databaseInfo.language),
},
start: localQueryInitialInfo.start,
id: localQueryInitialInfo.id,

5
extensions/ql-vscode/src/query-history/store/query-history-local-query-dto-mapper.ts
View File

@@ -20,6 +20,7 @@ import {
SortDirection,
SortedResultSetInfo,
} from "../../common/interface-types";
import { mapQueryLanguageToDomainModel } from "./query-history-dto-mapper";
export function mapLocalQueryItemToDomainModel(
localQuery: QueryHistoryLocalQueryDto,
@@ -82,6 +83,10 @@ function mapInitialQueryInfoToDomainModel(
databaseInfo: {
databaseUri: initialInfo.databaseInfo.databaseUri,
name: initialInfo.databaseInfo.name,
language:
initialInfo.databaseInfo.language === undefined
? undefined
: mapQueryLanguageToDomainModel(initialInfo.databaseInfo.language),
},
start: new Date(initialInfo.start),
id: initialInfo.id,

3
extensions/ql-vscode/src/query-history/store/query-history-local-query-dto.ts
View File

@@ -1,6 +1,8 @@
// Contains models and consts for the data we want to store in the query history store.
// Changes to these models should be done carefully and account for backwards compatibility of data.
import { QueryLanguageDto } from "./query-history-dto";
export interface QueryHistoryLocalQueryDto {
initialInfo: InitialQueryInfoDto;
t: "local";
@@ -27,6 +29,7 @@ export interface InitialQueryInfoDto {
interface DatabaseInfoDto {
name: string;
databaseUri: string;
language?: QueryLanguageDto;
}
interface PositionDto {

26
extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-domain-mapper.ts
View File

@@ -1,6 +1,5 @@
import {
QueryHistoryVariantAnalysisDto,
QueryLanguageDto,
QueryStatusDto,
VariantAnalysisDto,
VariantAnalysisFailureReasonDto,
@@ -22,9 +21,9 @@ import {
VariantAnalysisStatus,
} from "../../variant-analysis/shared/variant-analysis";
import { assertNever } from "../../common/helpers-pure";
import { QueryLanguage } from "../../common/query-language";
import { QueryStatus } from "../query-status";
import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
import { mapQueryLanguageToDto } from "./query-history-domain-mapper";
export function mapQueryHistoryVariantAnalysisToDto(
item: VariantAnalysisHistoryItem,
@@ -199,29 +198,6 @@ function mapVariantAnalysisStatusToDto(
}
}
function mapQueryLanguageToDto(language: QueryLanguage): QueryLanguageDto {
switch (language) {
case QueryLanguage.CSharp:
return QueryLanguageDto.CSharp;
case QueryLanguage.Cpp:
return QueryLanguageDto.Cpp;
case QueryLanguage.Go:
return QueryLanguageDto.Go;
case QueryLanguage.Java:
return QueryLanguageDto.Java;
case QueryLanguage.Javascript:
return QueryLanguageDto.Javascript;
case QueryLanguage.Python:
return QueryLanguageDto.Python;
case QueryLanguage.Ruby:
return QueryLanguageDto.Ruby;
case QueryLanguage.Swift:
return QueryLanguageDto.Swift;
default:
assertNever(language);
}
}
function mapQueryStatusToDto(status: QueryStatus): QueryStatusDto {
switch (status) {
case QueryStatus.InProgress:

28
extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto-mapper.ts
View File

@@ -1,6 +1,5 @@
import {
QueryHistoryVariantAnalysisDto,
QueryLanguageDto,
QueryStatusDto,
VariantAnalysisDto,
VariantAnalysisFailureReasonDto,
@@ -22,9 +21,9 @@ import {
VariantAnalysisStatus,
} from "../../variant-analysis/shared/variant-analysis";
import { assertNever } from "../../common/helpers-pure";
import { QueryLanguage } from "../../common/query-language";
import { QueryStatus } from "../query-status";
import { VariantAnalysisHistoryItem } from "../variant-analysis-history-item";
import { mapQueryLanguageToDomainModel } from "./query-history-dto-mapper";
export function mapQueryHistoryVariantAnalysisToDomainModel(
item: QueryHistoryVariantAnalysisDto,
@@ -215,31 +214,6 @@ function mapVariantAnalysisStatusToDomainModel(
}
}
function mapQueryLanguageToDomainModel(
language: QueryLanguageDto,
): QueryLanguage {
switch (language) {
case QueryLanguageDto.CSharp:
return QueryLanguage.CSharp;
case QueryLanguageDto.Cpp:
return QueryLanguage.Cpp;
case QueryLanguageDto.Go:
return QueryLanguage.Go;
case QueryLanguageDto.Java:
return QueryLanguage.Java;
case QueryLanguageDto.Javascript:
return QueryLanguage.Javascript;
case QueryLanguageDto.Python:
return QueryLanguage.Python;
case QueryLanguageDto.Ruby:
return QueryLanguage.Ruby;
case QueryLanguageDto.Swift:
return QueryLanguage.Swift;
default:
assertNever(language);
}
}
function mapQueryStatusToDomainModel(status: QueryStatusDto): QueryStatus {
switch (status) {
case QueryStatusDto.InProgress:

13
extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto.ts
View File

@@ -1,6 +1,8 @@
// Contains models and consts for the data we want to store in the query history store.
// Changes to these models should be done carefully and account for backwards compatibility of data.
import { QueryLanguageDto } from "./query-history-dto";
export interface QueryHistoryVariantAnalysisDto {
readonly t: "variant-analysis";
failureReason?: string;
@@ -97,17 +99,6 @@ export enum VariantAnalysisStatusDto {
Canceled = "canceled",
}
export enum QueryLanguageDto {
CSharp = "csharp",
Cpp = "cpp",
Go = "go",
Java = "java",
Javascript = "javascript",
Python = "python",
Ruby = "ruby",
Swift = "swift",
}
export enum QueryStatusDto {
InProgress = "InProgress",
Completed = "Completed",

1
extensions/ql-vscode/src/query-server/server-process.ts
View File

@@ -26,6 +26,7 @@ export class ServerProcess implements Disposable {
this.connection.end();
this.child.stdin!.end();
this.child.stderr!.destroy();
this.child.removeAllListeners();
// TODO kill the process if it doesn't terminate after a certain time limit.
// On Windows, we usually have to terminate the process before closing its stdout.

19
extensions/ql-vscode/src/stories/method-modeling/MethodModeling.stories.tsx
View File

@@ -3,6 +3,7 @@ import * as React from "react";
import { Meta, StoryFn } from "@storybook/react";
import { MethodModeling as MethodModelingComponent } from "../../view/method-modeling/MethodModeling";
import { createMethod } from "../../../test/factories/model-editor/method-factories";
export default {
title: "Method Modeling/Method Modeling",
component: MethodModelingComponent,
@@ -12,11 +13,23 @@ const Template: StoryFn<typeof MethodModelingComponent> = (args) => (
<MethodModelingComponent {...args} />
);
const method = createMethod();
export const MethodUnmodeled = Template.bind({});
MethodUnmodeled.args = { modelingStatus: "unmodeled" };
MethodUnmodeled.args = {
method,
modelingStatus: "unmodeled",
};
export const MethodModeled = Template.bind({});
MethodModeled.args = { modelingStatus: "unsaved" };
MethodModeled.args = {
method,
modelingStatus: "unsaved",
};
export const MethodSaved = Template.bind({});
MethodSaved.args = { modelingStatus: "saved" };
MethodSaved.args = {
method,
modelingStatus: "saved",
};

53
extensions/ql-vscode/src/stories/method-modeling/MethodModelingInputs.stories.tsx Normal file
View File

@@ -0,0 +1,53 @@
import * as React from "react";
import { Meta, StoryFn } from "@storybook/react";
import { MethodModelingInputs as MethodModelingInputsComponent } from "../../view/method-modeling/MethodModelingInputs";
import { createMethod } from "../../../test/factories/model-editor/method-factories";
import { createModeledMethod } from "../../../test/factories/model-editor/modeled-method-factories";
import { useState } from "react";
import { ModeledMethod } from "../../model-editor/modeled-method";
export default {
title: "Method Modeling/Method Modeling Inputs",
component: MethodModelingInputsComponent,
argTypes: {
modeledMethod: {
control: {
disable: true,
},
},
},
} as Meta<typeof MethodModelingInputsComponent>;
const Template: StoryFn<typeof MethodModelingInputsComponent> = (args) => {
const [m, setModeledMethod] = useState<ModeledMethod | undefined>(
args.modeledMethod,
);
const onChange = (modeledMethod: ModeledMethod) => {
setModeledMethod(modeledMethod);
};
return (
<MethodModelingInputsComponent
{...args}
modeledMethod={m}
onChange={onChange}
/>
);
};
const method = createMethod();
const modeledMethod = createModeledMethod();
export const UnmodeledMethod = Template.bind({});
UnmodeledMethod.args = {
method,
};
export const FullyModeledMethod = Template.bind({});
FullyModeledMethod.args = {
method,
modeledMethod,
};

1
extensions/ql-vscode/src/stories/model-editor/LibraryRow.stories.tsx
View File

@@ -214,6 +214,7 @@ LibraryRow.args = {
extensionPack: createMockExtensionPack(),
showFlowGeneration: true,
showLlmButton: true,
showMultipleModels: true,
mode: Mode.Application,
},
hideModeledMethods: false,

2
extensions/ql-vscode/src/stories/model-editor/MethodName.stories.tsx
View File

@@ -3,7 +3,7 @@ import * as React from "react";
import { Meta, StoryFn } from "@storybook/react";
import { MethodName as MethodNameComponent } from "../../view/model-editor/MethodName";
import { createMethod } from "../../../test/factories/data-extension/method-factories";
import { createMethod } from "../../../test/factories/model-editor/method-factories";
export default {
title: "CodeQL Model Editor/Method Name",

8
extensions/ql-vscode/src/stories/model-editor/MethodRow.stories.tsx
View File

@@ -27,7 +27,7 @@ const method: Method = {
methodName: "open",
methodParameters: "()",
supported: false,
supportedType: "summary",
supportedType: "none",
usages: [
{
label: "open(...)",
@@ -70,30 +70,35 @@ export const Unmodeled = Template.bind({});
Unmodeled.args = {
method,
modeledMethod: undefined,
methodCanBeModeled: true,
};
export const Source = Template.bind({});
Source.args = {
method,
modeledMethod: { ...modeledMethod, type: "source" },
methodCanBeModeled: true,
};
export const Sink = Template.bind({});
Sink.args = {
method,
modeledMethod: { ...modeledMethod, type: "sink" },
methodCanBeModeled: true,
};
export const Summary = Template.bind({});
Summary.args = {
method,
modeledMethod: { ...modeledMethod, type: "summary" },
methodCanBeModeled: true,
};
export const Neutral = Template.bind({});
Neutral.args = {
method,
modeledMethod: { ...modeledMethod, type: "neutral" },
methodCanBeModeled: true,
};
export const AlreadyModeled = Template.bind({});
@@ -107,4 +112,5 @@ ModelingInProgress.args = {
method,
modeledMethod,
modelingInProgress: true,
methodCanBeModeled: true,
};

1
extensions/ql-vscode/src/stories/model-editor/ModelEditor.stories.tsx
View File

@@ -30,6 +30,7 @@ ModelEditor.args = {
},
showFlowGeneration: true,
showLlmButton: true,
showMultipleModels: true,
mode: Mode.Application,
},
initialMethods: [

33
extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
View File

@@ -37,15 +37,7 @@ import {
import { QueryLanguage } from "../common/query-language";
import { tryGetQueryMetadata } from "../codeql-cli/query-metadata";
import { askForLanguage, findLanguage } from "../codeql-cli/query-language";
interface QlPack {
name: string;
version: string;
library?: boolean;
dependencies: { [key: string]: string };
defaultSuite?: Array<Record<string, unknown>>;
defaultSuiteFile?: string;
}
import { QlPackFile } from "../packaging/qlpack-file";
/**
* Well-known names for the query pack used by the server.
@@ -395,7 +387,7 @@ async function fixPackFile(
)} file in '${queryPackDir}'`,
);
}
const qlpack = load(await readFile(packPath, "utf8")) as QlPack;
const qlpack = load(await readFile(packPath, "utf8")) as QlPackFile;
updateDefaultSuite(qlpack, packRelativePath);
removeWorkspaceRefs(qlpack);
@@ -416,7 +408,11 @@ async function injectExtensionPacks(
)} file in '${queryPackDir}'`,
);
}
const syntheticQueryPack = load(await readFile(qlpackFile, "utf8")) as QlPack;
const syntheticQueryPack = load(
await readFile(qlpackFile, "utf8"),
) as QlPackFile;
const dependencies = syntheticQueryPack.dependencies ?? {};
const extensionPacks = await cliServer.resolveQlpacks(workspaceFolders, true);
Object.entries(extensionPacks).forEach(([name, paths]) => {
@@ -433,13 +429,16 @@ async function injectExtensionPacks(
// Add this extension pack as a dependency. It doesn't matter which
// version we specify, since we are guaranteed that the extension pack
// is resolved from source at the given path.
syntheticQueryPack.dependencies[name] = "*";
dependencies[name] = "*";
});
syntheticQueryPack.dependencies = dependencies;
await writeFile(qlpackFile, dump(syntheticQueryPack));
await cliServer.clearCache();
}
function updateDefaultSuite(qlpack: QlPack, packRelativePath: string) {
function updateDefaultSuite(qlpack: QlPackFile, packRelativePath: string) {
delete qlpack.defaultSuiteFile;
qlpack.defaultSuite = generateDefaultSuite(packRelativePath);
}
@@ -541,8 +540,12 @@ async function getControllerRepoFromApi(
}
}
export function removeWorkspaceRefs(qlpack: QlPack) {
for (const [key, value] of Object.entries(qlpack.dependencies || {})) {
export function removeWorkspaceRefs(qlpack: QlPackFile) {
if (!qlpack.dependencies) {
return;
}
for (const [key, value] of Object.entries(qlpack.dependencies)) {
if (value === "${workspace}") {
qlpack.dependencies[key] = "*";
}

17
extensions/ql-vscode/src/view/common/RawNumberValue.tsx Normal file
View File

@@ -0,0 +1,17 @@
import * as React from "react";
import { styled } from "styled-components";
import { formatDecimal } from "../../common/number";
const RightAlignedSpan = styled.span`
display: inline-block;
text-align: right;
width: 100%;
`;
type Props = {
value: number;
};
export const RawNumberValue = ({ value }: Props) => {
return <RightAlignedSpan>{formatDecimal(value)}</RightAlignedSpan>;
};

45
extensions/ql-vscode/src/view/method-modeling/MethodModeling.tsx
View File

@@ -1,44 +1,71 @@
import * as React from "react";
import { styled } from "styled-components";
import {
ModelingStatus,
ModelingStatusIndicator,
} from "../model-editor/ModelingStatusIndicator";
import { ModelingStatus } from "../../model-editor/shared/modeling-status";
import { ModelingStatusIndicator } from "../model-editor/ModelingStatusIndicator";
import { Method } from "../../model-editor/method";
import { MethodName } from "../model-editor/MethodName";
import { ModeledMethod } from "../../model-editor/modeled-method";
import { MethodModelingInputs } from "./MethodModelingInputs";
import { VSCodeTag } from "@vscode/webview-ui-toolkit/react";
import { ReviewInEditorButton } from "./ReviewInEditorButton";
const Container = styled.div`
background-color: var(--vscode-peekViewResult-background);
padding: 0.3rem;
margin-bottom: 1rem;
width: 100%;
`;
const Title = styled.div`
padding-bottom: 0.3rem;
font-size: 1.2em;
font-size: 0.7rem;
text-transform: uppercase;
display: flex;
justify-content: space-between;
`;
const DependencyContainer = styled.div`
display: flex;
justify-content: space-between;
flex-direction: row;
align-items: center;
gap: 0.5em;
background-color: var(--vscode-editor-background);
border: 0.05rem solid var(--vscode-panelSection-border);
border-radius: 0.3rem;
padding: 0.5rem;
word-wrap: break-word;
word-break: break-all;
`;
export type MethodModelingProps = {
modelingStatus: ModelingStatus;
method: Method;
modeledMethod: ModeledMethod | undefined;
onChange: (modeledMethod: ModeledMethod) => void;
};
export const MethodModeling = ({
modelingStatus,
modeledMethod,
method,
onChange,
}: MethodModelingProps): JSX.Element => {
return (
<Container>
<Title>API or Method</Title>
<Title>
{method.packageName}
{method.libraryVersion && <>@{method.libraryVersion}</>}
{modelingStatus === "unsaved" ? <VSCodeTag>Unsaved</VSCodeTag> : null}
</Title>
<DependencyContainer>
<MethodName {...method} />
<ModelingStatusIndicator status={modelingStatus} />
<MethodName {...method} />
</DependencyContainer>
<MethodModelingInputs
method={method}
modeledMethod={modeledMethod}
onChange={onChange}
/>
<ReviewInEditorButton method={method} />
</Container>
);
};

66
extensions/ql-vscode/src/view/method-modeling/MethodModelingInputs.tsx Normal file
View File

@@ -0,0 +1,66 @@
import * as React from "react";
import { styled } from "styled-components";
import { Method } from "../../model-editor/method";
import { ModeledMethod } from "../../model-editor/modeled-method";
import { ModelTypeDropdown } from "../model-editor/ModelTypeDropdown";
import { ModelInputDropdown } from "../model-editor/ModelInputDropdown";
import { ModelOutputDropdown } from "../model-editor/ModelOutputDropdown";
import { ModelKindDropdown } from "../model-editor/ModelKindDropdown";
const Container = styled.div`
padding-top: 0.5rem;
`;
const Input = styled.label``;
const Name = styled.span`
display: block;
padding-bottom: 0.3rem;
`;
export type MethodModelingInputsProps = {
method: Method;
modeledMethod: ModeledMethod | undefined;
onChange: (modeledMethod: ModeledMethod) => void;
};
export const MethodModelingInputs = ({
method,
modeledMethod,
onChange,
}: MethodModelingInputsProps): JSX.Element => {
const inputProps = {
method,
modeledMethod,
onChange,
};
return (
<>
<Container>
<Input>
<Name>Model Type</Name>
<ModelTypeDropdown {...inputProps} />
</Input>
</Container>
<Container>
<Input>
<Name>Input</Name>
<ModelInputDropdown {...inputProps} />
</Input>
</Container>
<Container>
<Input>
<Name>Output</Name>
<ModelOutputDropdown {...inputProps} />
</Input>
</Container>
<Container>
<Input>
<Name>Kind</Name>
<ModelKindDropdown {...inputProps} />
</Input>
</Container>
</>
);
};

55
extensions/ql-vscode/src/view/method-modeling/MethodModelingView.tsx
View File

@@ -1,22 +1,48 @@
import * as React from "react";
import { useEffect, useState } from "react";
import { useEffect, useMemo, useState } from "react";
import { MethodModeling } from "./MethodModeling";
import { ModelingStatus } from "../model-editor/ModelingStatusIndicator";
import { getModelingStatus } from "../../model-editor/shared/modeling-status";
import { Method } from "../../model-editor/method";
import { ToMethodModelingMessage } from "../../common/interface-types";
import { assertNever } from "../../common/helpers-pure";
import { ModeledMethod } from "../../model-editor/modeled-method";
import { vscode } from "../vscode-api";
export function MethodModelingView(): JSX.Element {
const [method, setMethod] = useState<Method | undefined>(undefined);
const [modeledMethod, setModeledMethod] = React.useState<
ModeledMethod | undefined
>(undefined);
const [isMethodModified, setIsMethodModified] = useState<boolean>(false);
const modelingStatus = useMemo(
() => getModelingStatus(modeledMethod, isMethodModified),
[modeledMethod, isMethodModified],
);
useEffect(() => {
const listener = (evt: MessageEvent) => {
if (evt.origin === window.origin) {
const msg: ToMethodModelingMessage = evt.data;
if (msg.t === "setMethod") {
setMethod(msg.method);
} else {
assertNever(msg.t);
switch (msg.t) {
case "setMethod":
setMethod(msg.method);
break;
case "setModeledMethod":
setModeledMethod(msg.method);
break;
case "setMethodModified":
setIsMethodModified(msg.isModified);
break;
case "setSelectedMethod":
setMethod(msg.method);
setModeledMethod(msg.modeledMethod);
setIsMethodModified(msg.isModified);
break;
default:
assertNever(msg);
}
} else {
// sanitize origin
@@ -35,6 +61,19 @@ export function MethodModelingView(): JSX.Element {
return <>Select method to model</>;
}
const modelingStatus: ModelingStatus = "saved";
return <MethodModeling modelingStatus={modelingStatus} method={method} />;
const onChange = (modeledMethod: ModeledMethod) => {
vscode.postMessage({
t: "setModeledMethod",
method: modeledMethod,
});
};
return (
<MethodModeling
modelingStatus={modelingStatus}
method={method}
modeledMethod={modeledMethod}
onChange={onChange}
/>
);
}

25
extensions/ql-vscode/src/view/method-modeling/ReviewInEditorButton.tsx Normal file
View File

@@ -0,0 +1,25 @@
import * as React from "react";
import { useCallback } from "react";
import { styled } from "styled-components";
import { vscode } from "../vscode-api";
import TextButton from "../common/TextButton";
import { Method } from "../../model-editor/method";
const Button = styled(TextButton)`
margin-top: 0.5rem;
`;
type Props = {
method: Method;
};
export const ReviewInEditorButton = ({ method }: Props) => {
const handleClick = useCallback(() => {
vscode.postMessage({
t: "revealInModelEditor",
method,
});
}, [method]);
return <Button onClick={handleClick}>Review in editor</Button>;
};

15
extensions/ql-vscode/src/view/method-modeling/__tests__/MethodModeling.spec.tsx
View File

@@ -1,18 +1,27 @@
import * as React from "react";
import { render as reactRender, screen } from "@testing-library/react";
import { MethodModeling, MethodModelingProps } from "../MethodModeling";
import { createMethod } from "../../../../test/factories/data-extension/method-factories";
import { createMethod } from "../../../../test/factories/model-editor/method-factories";
import { createModeledMethod } from "../../../../test/factories/model-editor/modeled-method-factories";
describe(MethodModeling.name, () => {
const render = (props: MethodModelingProps) =>
reactRender(<MethodModeling {...props} />);
it("renders method modeling panel", () => {
const method = createMethod();
const modeledMethod = createModeledMethod();
const onChange = jest.fn();
render({
modelingStatus: "saved",
method: createMethod(),
method,
modeledMethod,
onChange,
});
expect(screen.getByText("API or Method")).toBeInTheDocument();
expect(
screen.getByText(`${method.packageName}@${method.libraryVersion}`),
).toBeInTheDocument();
});
});

100
extensions/ql-vscode/src/view/method-modeling/__tests__/MethodModelingInputs.spec.tsx Normal file
View File

@@ -0,0 +1,100 @@
import * as React from "react";
import { render as reactRender, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import {
MethodModelingInputs,
MethodModelingInputsProps,
} from "../MethodModelingInputs";
import { createMethod } from "../../../../test/factories/model-editor/method-factories";
import { createModeledMethod } from "../../../../test/factories/model-editor/modeled-method-factories";
describe(MethodModelingInputs.name, () => {
const render = (props: MethodModelingInputsProps) =>
reactRender(<MethodModelingInputs {...props} />);
const method = createMethod();
const modeledMethod = createModeledMethod();
const onChange = jest.fn();
it("renders the method modeling inputs", () => {
render({
method,
modeledMethod,
onChange,
});
// Check that all the labels are rendered.
expect(screen.getByText("Model Type")).toBeInTheDocument();
expect(screen.getByText("Input")).toBeInTheDocument();
expect(screen.getByText("Output")).toBeInTheDocument();
expect(screen.getByText("Kind")).toBeInTheDocument();
// Check that all the dropdowns are rendered.
const comboboxes = screen.getAllByRole("combobox");
expect(comboboxes.length).toBe(4);
const modelTypeDropdown = screen.getByRole("combobox", {
name: "Model type",
});
expect(modelTypeDropdown).toHaveValue("sink");
const modelTypeOptions = modelTypeDropdown.querySelectorAll("option");
expect(modelTypeOptions.length).toBe(5);
});
it("allows changing the type", async () => {
render({
method,
modeledMethod,
onChange,
});
const modelTypeDropdown = screen.getByRole("combobox", {
name: "Model type",
});
await userEvent.selectOptions(modelTypeDropdown, "source");
expect(onChange).toHaveBeenCalledWith(
expect.objectContaining({
type: "source",
}),
);
});
it("sets other dropdowns when model type is changed", () => {
const { rerender } = render({
method,
modeledMethod,
onChange,
});
const updatedModeledMethod = createModeledMethod({
type: "source",
});
rerender(
<MethodModelingInputs
method={method}
modeledMethod={updatedModeledMethod}
onChange={onChange}
/>,
);
const modelTypeDropdown = screen.getByRole("combobox", {
name: "Model type",
});
const modelInputDropdown = screen.getByRole("combobox", {
name: "Input",
});
const modelOutputDropdown = screen.getByRole("combobox", {
name: "Output",
});
const modelKindDropdown = screen.getByRole("combobox", {
name: "Kind",
});
expect(modelTypeDropdown).toHaveValue("source");
expect(modelInputDropdown).toHaveValue("-");
expect(modelOutputDropdown).toHaveValue("ReturnValue");
expect(modelKindDropdown).toHaveValue("local");
});
});

Some files were not shown because too many files have changed in this diff Show More