mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into starcke/language-context-store

Browse Source
This commit is contained in:
Anders Starcke Henriksen
2023-09-28 14:08:26 +02:00
parent bb25874299 d5c78fd67b
commit c7e5922bd5
20 changed files with 692 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
extensions/ql-vscode/package.json
View File

@@ -757,38 +757,74 @@
"command": "codeQLDatabases.displayAllLanguages",
"title": "All languages"
},
{
"command": "codeQLDatabases.displayAllLanguagesSelected",
"title": "All languages (selected)"
},
{
"command": "codeQLDatabases.displayCpp",
"title": "C/C++"
},
{
"command": "codeQLDatabases.displayCppSelected",
"title": "C/C++ (selected)"
},
{
"command": "codeQLDatabases.displayCsharp",
"title": "C#"
},
{
"command": "codeQLDatabases.displayCsharpSelected",
"title": "C# (selected)"
},
{
"command": "codeQLDatabases.displayGo",
"title": "Go"
},
{
"command": "codeQLDatabases.displayGoSelected",
"title": "Go (selected)"
},
{
"command": "codeQLDatabases.displayJava",
"title": "Java/Kotlin"
},
{
"command": "codeQLDatabases.displayJavaSelected",
"title": "Java/Kotlin (selected)"
},
{
"command": "codeQLDatabases.displayJavascript",
"title": "JavaScript/TypeScript"
},
{
"command": "codeQLDatabases.displayJavascriptSelected",
"title": "JavaScript/TypeScript (selected)"
},
{
"command": "codeQLDatabases.displayPython",
"title": "Python"
},
{
"command": "codeQLDatabases.displayPythonSelected",
"title": "Python (selected)"
},
{
"command": "codeQLDatabases.displayRuby",
"title": "Ruby"
},
{
"command": "codeQLDatabases.displayRubySelected",
"title": "Ruby (selected)"
},
{
"command": "codeQLDatabases.displaySwift",
"title": "Swift"
},
{
"command": "codeQLDatabases.displaySwiftSelected",
"title": "Swift (selected)"
},
{
"command": "codeQL.chooseDatabaseFolder",
"title": "CodeQL: Choose Database from Folder"
@@ -1568,38 +1604,74 @@
"command": "codeQLDatabases.displayAllLanguages",
"when": "false"
},
{
"command": "codeQLDatabases.displayAllLanguagesSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayCpp",
"when": "false"
},
{
"command": "codeQLDatabases.displayCppSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayCsharp",
"when": "false"
},
{
"command": "codeQLDatabases.displayCsharpSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayGo",
"when": "false"
},
{
"command": "codeQLDatabases.displayGoSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayJava",
"when": "false"
},
{
"command": "codeQLDatabases.displayJavaSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayJavascript",
"when": "false"
},
{
"command": "codeQLDatabases.displayJavascriptSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayPython",
"when": "false"
},
{
"command": "codeQLDatabases.displayPythonSelected",
"when": "false"
},
{
"command": "codeQLDatabases.displayRuby",
"when": "false"
},
{
"command": "codeQLDatabases.displayRubySelected",
"when": "false"
},
{
"command": "codeQLDatabases.displaySwift",
"when": "false"
},
{
"command": "codeQLDatabases.displaySwiftSelected",
"when": "false"
},
{
"command": "codeQLQueryHistory.openQueryContextMenu",
"when": "false"
@@ -1797,31 +1869,76 @@
],
"codeQLDatabases.languages": [
{
"command": "codeQLDatabases.displayAllLanguages"
"command": "codeQLDatabases.displayAllLanguages",
"when": "codeQLDatabases.languageFilter != All"
},
{
"command": "codeQLDatabases.displayCpp"
"command": "codeQLDatabases.displayAllLanguagesSelected",
"when": "codeQLDatabases.languageFilter == All"
},
{
"command": "codeQLDatabases.displayCsharp"
"command": "codeQLDatabases.displayCpp",
"when": "codeQLDatabases.languageFilter != cpp"
},
{
"command": "codeQLDatabases.displayGo"
"command": "codeQLDatabases.displayCppSelected",
"when": "codeQLDatabases.languageFilter == cpp"
},
{
"command": "codeQLDatabases.displayJava"
"command": "codeQLDatabases.displayCsharp",
"when": "codeQLDatabases.languageFilter != csharp"
},
{
"command": "codeQLDatabases.displayJavascript"
"command": "codeQLDatabases.displayCsharpSelected",
"when": "codeQLDatabases.languageFilter == csharp"
},
{
"command": "codeQLDatabases.displayPython"
"command": "codeQLDatabases.displayGo",
"when": "codeQLDatabases.languageFilter != go"
},
{
"command": "codeQLDatabases.displayRuby"
"command": "codeQLDatabases.displayGoSelected",
"when": "codeQLDatabases.languageFilter == go"
},
{
"command": "codeQLDatabases.displaySwift"
"command": "codeQLDatabases.displayJava",
"when": "codeQLDatabases.languageFilter != java"
},
{
"command": "codeQLDatabases.displayJavaSelected",
"when": "codeQLDatabases.languageFilter == java"
},
{
"command": "codeQLDatabases.displayJavascript",
"when": "codeQLDatabases.languageFilter != javascript"
},
{
"command": "codeQLDatabases.displayJavascriptSelected",
"when": "codeQLDatabases.languageFilter == javascript"
},
{
"command": "codeQLDatabases.displayPython",
"when": "codeQLDatabases.languageFilter != python"
},
{
"command": "codeQLDatabases.displayPythonSelected",
"when": "codeQLDatabases.languageFilter == python"
},
{
"command": "codeQLDatabases.displayRuby",
"when": "codeQLDatabases.languageFilter != ruby"
},
{
"command": "codeQLDatabases.displayRubySelected",
"when": "codeQLDatabases.languageFilter == ruby"
},
{
"command": "codeQLDatabases.displaySwift",
"when": "codeQLDatabases.languageFilter != swift"
},
{
"command": "codeQLDatabases.displaySwiftSelected",
"when": "codeQLDatabases.languageFilter == swift"
}
]
},

111
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -6,7 +6,6 @@ import { dirname, join, delimiter } from "path";
import * as sarif from "sarif";
import { SemVer } from "semver";
import { Readable } from "stream";
import { StringDecoder } from "string_decoder";
import tk from "tree-kill";
import { promisify } from "util";
import { CancellationToken, Disposable, Uri } from "vscode";
@@ -31,6 +30,7 @@ import { CompilationMessage } from "../query-server/legacy-messages";
import { sarifParser } from "../common/sarif-parser";
import { App } from "../common/app";
import { QueryLanguage } from "../common/query-language";
import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
/**
* The version of the SARIF format that we are using.
@@ -1649,120 +1649,13 @@ export async function runCodeQlCliCommand(
}
}
/**
* Buffer to hold state used when splitting a text stream into lines.
*/
class SplitBuffer {
private readonly decoder = new StringDecoder("utf8");
private readonly maxSeparatorLength: number;
private buffer = "";
private searchIndex = 0;
constructor(private readonly separators: readonly string[]) {
this.maxSeparatorLength = separators
.map((s) => s.length)
.reduce((a, b) => Math.max(a, b), 0);
}
/**
* Append new text data to the buffer.
* @param chunk The chunk of data to append.
*/
public addChunk(chunk: Buffer): void {
this.buffer += this.decoder.write(chunk);
}
/**
* Signal that the end of the input stream has been reached.
*/
public end(): void {
this.buffer += this.decoder.end();
this.buffer += this.separators[0]; // Append a separator to the end to ensure the last line is returned.
}
/**
* A version of startsWith that isn't overriden by a broken version of ms-python.
*
* The definition comes from
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
* which is CC0/public domain
*
* See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
*/
private static startsWith(
s: string,
searchString: string,
position: number,
): boolean {
const pos = position > 0 ? position | 0 : 0;
return s.substring(pos, pos + searchString.length) === searchString;
}
/**
* Extract the next full line from the buffer, if one is available.
* @returns The text of the next available full line (without the separator), or `undefined` if no
* line is available.
*/
public getNextLine(): string | undefined {
while (this.searchIndex <= this.buffer.length - this.maxSeparatorLength) {
for (const separator of this.separators) {
if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
const line = this.buffer.slice(0, this.searchIndex);
this.buffer = this.buffer.slice(this.searchIndex + separator.length);
this.searchIndex = 0;
return line;
}
}
this.searchIndex++;
}
return undefined;
}
}
/**
* Splits a text stream into lines based on a list of valid line separators.
* @param stream The text stream to split. This stream will be fully consumed.
* @param separators The list of strings that act as line separators.
* @returns A sequence of lines (not including separators).
*/
async function* splitStreamAtSeparators(
stream: Readable,
separators: string[],
): AsyncGenerator<string, void, unknown> {
const buffer = new SplitBuffer(separators);
for await (const chunk of stream) {
buffer.addChunk(chunk);
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
buffer.end();
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
/**
* Standard line endings for splitting human-readable text.
*/
const lineEndings = ["\r\n", "\r", "\n"];
/**
* Log a text stream to a `Logger` interface.
* @param stream The stream to log.
* @param logger The logger that will consume the stream output.
*/
async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
for await (const line of splitStreamAtSeparators(stream, lineEndings)) {
for await (const line of splitStreamAtSeparators(stream, LINE_ENDINGS)) {
// Await the result of log here in order to ensure the logs are written in the correct order.
await logger.log(line);
}

9
extensions/ql-vscode/src/common/commands.ts
View File

@@ -228,6 +228,15 @@ export type LocalDatabasesCommands = {
"codeQLDatabases.displayPython": () => Promise<void>;
"codeQLDatabases.displayRuby": () => Promise<void>;
"codeQLDatabases.displaySwift": () => Promise<void>;
"codeQLDatabases.displayAllLanguagesSelected": () => Promise<void>;
"codeQLDatabases.displayCppSelected": () => Promise<void>;
"codeQLDatabases.displayCsharpSelected": () => Promise<void>;
"codeQLDatabases.displayGoSelected": () => Promise<void>;
"codeQLDatabases.displayJavaSelected": () => Promise<void>;
"codeQLDatabases.displayJavascriptSelected": () => Promise<void>;
"codeQLDatabases.displayPythonSelected": () => Promise<void>;
"codeQLDatabases.displayRubySelected": () => Promise<void>;
"codeQLDatabases.displaySwiftSelected": () => Promise<void>;
// Database panel context menu
"codeQLDatabases.setCurrentDatabase": (

125
extensions/ql-vscode/src/common/split-stream.ts Normal file
View File

@@ -0,0 +1,125 @@
import { Readable } from "stream";
import { StringDecoder } from "string_decoder";
/**
* Buffer to hold state used when splitting a text stream into lines.
*/
export class SplitBuffer {
private readonly decoder = new StringDecoder("utf8");
private readonly maxSeparatorLength: number;
private buffer = "";
private searchIndex = 0;
private ended = false;
constructor(private readonly separators: readonly string[]) {
this.maxSeparatorLength = separators
.map((s) => s.length)
.reduce((a, b) => Math.max(a, b), 0);
}
/**
* Append new text data to the buffer.
* @param chunk The chunk of data to append.
*/
public addChunk(chunk: Buffer): void {
this.buffer += this.decoder.write(chunk);
}
/**
* Signal that the end of the input stream has been reached.
*/
public end(): void {
this.buffer += this.decoder.end();
this.ended = true;
}
/**
* A version of startsWith that isn't overriden by a broken version of ms-python.
*
* The definition comes from
* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
* which is CC0/public domain
*
* See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
*/
private static startsWith(
s: string,
searchString: string,
position: number,
): boolean {
const pos = position > 0 ? position | 0 : 0;
return s.substring(pos, pos + searchString.length) === searchString;
}
/**
* Extract the next full line from the buffer, if one is available.
* @returns The text of the next available full line (without the separator), or `undefined` if no
* line is available.
*/
public getNextLine(): string | undefined {
// If we haven't received all of the input yet, don't search too close to the end of the buffer,
// or we could match a separator that's split across two chunks. For example, we could see "\r"
// at the end of the buffer and match that, even though we were about to receive a "\n" right
// after it.
const maxSearchIndex = this.ended
? this.buffer.length - 1
: this.buffer.length - this.maxSeparatorLength;
while (this.searchIndex <= maxSearchIndex) {
for (const separator of this.separators) {
if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
const line = this.buffer.slice(0, this.searchIndex);
this.buffer = this.buffer.slice(this.searchIndex + separator.length);
this.searchIndex = 0;
return line;
}
}
this.searchIndex++;
}
if (this.ended && this.buffer.length > 0) {
// If we still have some text left in the buffer, return it as the last line.
const line = this.buffer;
this.buffer = "";
this.searchIndex = 0;
return line;
} else {
return undefined;
}
}
}
/**
* Splits a text stream into lines based on a list of valid line separators.
* @param stream The text stream to split. This stream will be fully consumed.
* @param separators The list of strings that act as line separators.
* @returns A sequence of lines (not including separators).
*/
export async function* splitStreamAtSeparators(
stream: Readable,
separators: string[],
): AsyncGenerator<string, void, unknown> {
const buffer = new SplitBuffer(separators);
for await (const chunk of stream) {
buffer.addChunk(chunk);
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
buffer.end();
let line: string | undefined;
do {
line = buffer.getNextLine();
if (line !== undefined) {
yield line;
}
} while (line !== undefined);
}
/**
* Standard line endings for splitting human-readable text.
*/
export const LINE_ENDINGS = ["\r\n", "\r", "\n"];

15
extensions/ql-vscode/src/databases/code-search-api.ts
View File

@@ -3,10 +3,7 @@ import { throttling } from "@octokit/plugin-throttling";
import { Octokit } from "@octokit/rest";
import { Progress, CancellationToken } from "vscode";
import { Credentials } from "../common/authentication";
import {
NotificationLogger,
showAndLogWarningMessage,
} from "../common/logging";
import { BaseLogger } from "../common/logging";
export async function getCodeSearchRepositories(
query: string,
@@ -16,7 +13,7 @@ export async function getCodeSearchRepositories(
}>,
token: CancellationToken,
credentials: Credentials,
logger: NotificationLogger,
logger: BaseLogger,
): Promise<string[]> {
let nwos: string[] = [];
const octokit = await provideOctokitWithThrottling(credentials, logger);
@@ -47,7 +44,7 @@ export async function getCodeSearchRepositories(
async function provideOctokitWithThrottling(
credentials: Credentials,
logger: NotificationLogger,
logger: BaseLogger,
): Promise<Octokit> {
const MyOctokit = Octokit.plugin(throttling);
const auth = await credentials.getAccessToken();
@@ -57,16 +54,14 @@ async function provideOctokitWithThrottling(
retry,
throttle: {
onRateLimit: (retryAfter: number, options: any): boolean => {
void showAndLogWarningMessage(
logger,
void logger.log(
`Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`,
);
return true;
},
onSecondaryRateLimit: (_retryAfter: number, options: any): void => {
void showAndLogWarningMessage(
logger,
void logger.log(
`Secondary Rate Limit detected for request ${options.method} ${options.url}`,
);
},

30
extensions/ql-vscode/src/databases/local-databases-ui.ts
View File

@@ -296,6 +296,26 @@ export class DatabaseUI extends DisposableObject {
this,
QueryLanguage.Swift,
),
"codeQLDatabases.displayAllLanguagesSelected":
this.handleClearLanguageFilter.bind(this),
"codeQLDatabases.displayCppSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Cpp),
"codeQLDatabases.displayCsharpSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.CSharp),
"codeQLDatabases.displayGoSelected": this.handleChangeLanguageFilter.bind(
this,
QueryLanguage.Go,
),
"codeQLDatabases.displayJavaSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Java),
"codeQLDatabases.displayJavascriptSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Javascript),
"codeQLDatabases.displayPythonSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Python),
"codeQLDatabases.displayRubySelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Ruby),
"codeQLDatabases.displaySwiftSelected":
this.handleChangeLanguageFilter.bind(this, QueryLanguage.Swift),
"codeQLDatabases.removeDatabase": createMultiSelectionCommand(
this.handleRemoveDatabase.bind(this),
),
@@ -588,10 +608,20 @@ export class DatabaseUI extends DisposableObject {
private async handleClearLanguageFilter() {
this.languageContext.clearLanguageContext();
await this.app.commands.execute(
"setContext",
"codeQLDatabases.languageFilter",
"All",
);
}
private async handleChangeLanguageFilter(languageFilter: QueryLanguage) {
this.languageContext.setLanguageContext(languageFilter);
await this.app.commands.execute(
"setContext",
"codeQLDatabases.languageFilter",
languageFilter,
);
}
private async handleUpgradeCurrentDatabase(): Promise<void> {

2
extensions/ql-vscode/src/databases/ui/db-panel.ts
View File

@@ -409,7 +409,7 @@ export class DbPanel extends DisposableObject {
return;
}
void window.withProgress(
await window.withProgress(
{
location: ProgressLocation.Notification,
title: "Searching for repositories... This might take a while",

15
extensions/ql-vscode/src/local-queries/query-resolver.ts
View File

@@ -39,12 +39,14 @@ export interface QueryConstraints {
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param constraints Constraints on the queries to search for.
* @param additionalPacks Additional pack paths to search.
* @returns The found queries from the first pack in which any matching queries were found.
*/
async function resolveQueriesFromPacks(
export async function resolveQueriesFromPacks(
cli: CodeQLCliServer,
qlpacks: string[],
constraints: QueryConstraints,
additionalPacks: string[] = [],
): Promise<string[]> {
const suiteFile = (
await file({
@@ -67,10 +69,10 @@ async function resolveQueriesFromPacks(
"utf8",
);
return await cli.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
return await cli.resolveQueriesInSuite(suiteFile, [
...getOnDiskWorkspaceFolders(),
...additionalPacks,
]);
}
export async function resolveQueriesByLanguagePack(
@@ -97,6 +99,7 @@ export async function resolveQueriesByLanguagePack(
* @param packsToSearch The list of packs to search.
* @param name The name of the query to use in error messages.
* @param constraints Constraints on the queries to search for.
* @param additionalPacks Additional pack paths to search.
* @returns The found queries from the first pack in which any matching queries were found.
*/
export async function resolveQueries(
@@ -104,11 +107,13 @@ export async function resolveQueries(
packsToSearch: string[],
name: string,
constraints: QueryConstraints,
additionalPacks: string[] = [],
): Promise<string[]> {
const queries = await resolveQueriesFromPacks(
cli,
packsToSearch,
constraints,
additionalPacks,
);
if (queries.length > 0) {
return queries;

104
extensions/ql-vscode/src/log-insights/summary-parser.ts
View File

@@ -1,4 +1,5 @@
import { writeFile, promises } from "fs-extra";
import { createReadStream, writeFile } from "fs-extra";
import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
/**
* Location information for a single pipeline invocation in the RA.
@@ -64,59 +65,64 @@ export async function generateSummarySymbolsFile(
async function generateSummarySymbols(
summaryPath: string,
): Promise<SummarySymbols> {
const summary = await promises.readFile(summaryPath, {
const stream = createReadStream(summaryPath, {
encoding: "utf-8",
});
const symbols: SummarySymbols = {
predicates: {},
};
try {
const lines = splitStreamAtSeparators(stream, LINE_ENDINGS);
const lines = summary.split(/\r?\n/);
let lineNumber = 0;
while (lineNumber < lines.length) {
const startLineNumber = lineNumber;
lineNumber++;
const startLine = lines[startLineNumber];
const nonRecursiveMatch = startLine.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
let predicateName: string | undefined = undefined;
const symbols: SummarySymbols = {
predicates: {},
};
let lineNumber = 0;
let raStartLine = 0;
let iteration = 0;
if (nonRecursiveMatch) {
predicateName = nonRecursiveMatch.groups!.predicateName;
} else {
const recursiveMatch = startLine.match(RECURSIVE_TUPLE_COUNT_REGEXP);
if (recursiveMatch?.groups) {
predicateName = recursiveMatch.groups.predicateName;
iteration = parseInt(recursiveMatch.groups.iteration);
}
}
if (predicateName !== undefined) {
const raStartLine = lineNumber;
let raEndLine: number | undefined = undefined;
while (lineNumber < lines.length && raEndLine === undefined) {
const raLine = lines[lineNumber];
const returnMatch = raLine.match(RETURN_REGEXP);
let predicateName: string | undefined = undefined;
let startLine = 0;
for await (const line of lines) {
if (predicateName === undefined) {
// Looking for the start of the predicate.
const nonRecursiveMatch = line.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
if (nonRecursiveMatch) {
iteration = 0;
predicateName = nonRecursiveMatch.groups!.predicateName;
} else {
const recursiveMatch = line.match(RECURSIVE_TUPLE_COUNT_REGEXP);
if (recursiveMatch?.groups) {
predicateName = recursiveMatch.groups.predicateName;
iteration = parseInt(recursiveMatch.groups.iteration);
}
}
if (predicateName !== undefined) {
startLine = lineNumber;
raStartLine = lineNumber + 1;
}
} else {
const returnMatch = line.match(RETURN_REGEXP);
if (returnMatch) {
raEndLine = lineNumber;
}
lineNumber++;
}
if (raEndLine !== undefined) {
let symbol = symbols.predicates[predicateName];
if (symbol === undefined) {
symbol = {
iterations: {},
let symbol = symbols.predicates[predicateName];
if (symbol === undefined) {
symbol = {
iterations: {},
};
symbols.predicates[predicateName] = symbol;
}
symbol.iterations[iteration] = {
startLine,
raStartLine,
raEndLine: lineNumber,
};
symbols.predicates[predicateName] = symbol;
}
symbol.iterations[iteration] = {
startLine: lineNumber,
raStartLine,
raEndLine,
};
}
}
}
return symbols;
predicateName = undefined;
}
}
lineNumber++;
}
return symbols;
} finally {
stream.close();
}
}

13
extensions/ql-vscode/src/model-editor/auto-model-codeml-queries.ts
View File

@@ -7,7 +7,6 @@ import { Mode } from "./shared/mode";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { interpretResultsSarif } from "../query-results";
import { join } from "path";
import { assertNever } from "../common/helpers-pure";
import { dir } from "tmp-promise";
import { writeFile, outputFile } from "fs-extra";
import { dump as dumpYaml } from "js-yaml";
@@ -16,17 +15,7 @@ import { runQuery } from "../local-queries/run-query";
import { QueryMetadata } from "../common/interface-types";
import { CancellationTokenSource } from "vscode";
import { resolveQueries } from "../local-queries";
function modeTag(mode: Mode): string {
switch (mode) {
case Mode.Application:
return "application-mode";
case Mode.Framework:
return "framework-mode";
default:
assertNever(mode);
}
}
import { modeTag } from "./mode-tag";
type AutoModelQueriesOptions = {
mode: Mode;

27
extensions/ql-vscode/src/model-editor/external-api-usage-queries.ts
View File

@@ -16,6 +16,10 @@ import { fetchExternalApiQueries } from "./queries";
import { Method } from "./method";
import { runQuery } from "../local-queries/run-query";
import { decodeBqrsToMethods } from "./bqrs";
import {
resolveEndpointsQuery,
syntheticQueryPackName,
} from "./model-editor-queries";
type RunQueryOptions = {
cliServer: CodeQLCliServer;
@@ -88,7 +92,28 @@ export async function runExternalApiQueries(
await cliServer.resolveQlpacks(additionalPacks, true),
);
const queryPath = join(queryDir, queryNameFromMode(mode));
progress({
message: "Resolving query",
step: 2,
maxStep: externalApiQueriesProgressMaxStep,
});
// Resolve the queries from either codeql/java-queries or from the temporary queryDir
const queryPath = await resolveEndpointsQuery(
cliServer,
databaseItem.language,
mode,
[syntheticQueryPackName],
[queryDir],
);
if (!queryPath) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError`The ${mode} model editor query could not be found. Try re-opening the model editor. If that doesn't work, try upgrading the CodeQL libraries.`,
);
return;
}
// Run the actual query
const completedQuery = await runQuery({

13
extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts
View File

@@ -58,6 +58,7 @@ export class MethodModelingViewProvider
this.webviewView = webviewView;
this.setInitialState(webviewView);
this.registerToModelingStoreEvents();
}
@@ -72,6 +73,18 @@ export class MethodModelingViewProvider
}
}
private setInitialState(webviewView: vscode.WebviewView): void {
const selectedMethod = this.modelingStore.getSelectedMethodDetails();
if (selectedMethod) {
void webviewView.webview.postMessage({
t: "setSelectedMethod",
method: selectedMethod.method,
modeledMethod: selectedMethod.modeledMethod,
isModified: selectedMethod.isModified,
});
}
}
private async onMessage(msg: FromMethodModelingMessage): Promise<void> {
switch (msg.t) {
case "setModeledMethod": {

13
extensions/ql-vscode/src/model-editor/mode-tag.ts Normal file
View File

@@ -0,0 +1,13 @@
import { Mode } from "./shared/mode";
import { assertNever } from "../common/helpers-pure";
export function modeTag(mode: Mode): string {
switch (mode) {
case Mode.Application:
return "application-mode";
case Mode.Framework:
return "framework-mode";
default:
assertNever(mode);
}
}

1
extensions/ql-vscode/src/model-editor/model-editor-module.ts
View File

@@ -132,6 +132,7 @@ export class ModelEditorModule extends DisposableObject {
const { path: queryDir, cleanup: cleanupQueryDir } = await dir({
unsafeCleanup: true,
});
const success = await setUpPack(this.cliServer, queryDir, language);
if (!success) {
await cleanupQueryDir();

136
extensions/ql-vscode/src/model-editor/model-editor-queries.ts
View File

@@ -5,9 +5,27 @@ import { dump } from "js-yaml";
import { prepareExternalApiQuery } from "./external-api-usage-queries";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { showLlmGeneration } from "../config";
import { Mode } from "./shared/mode";
import { resolveQueriesFromPacks } from "../local-queries";
import { modeTag } from "./mode-tag";
export const syntheticQueryPackName = "codeql/external-api-usage";
/**
* setUpPack sets up a directory to use for the data extension editor queries.
* setUpPack sets up a directory to use for the data extension editor queries if required.
*
* There are two cases (example language is Java):
* - In case the queries are present in the codeql/java-queries, we don't need to write our own queries
* to disk. We still need to create a synthetic query pack so we can pass the queryDir to the query
* resolver without caring about whether the queries are present in the pack or not.
* - In case the queries are not present in the codeql/java-queries, we need to write our own queries
* to disk. We will create a synthetic query pack and install its dependencies so it is fully independent
* and we can simply pass it through when resolving the queries.
*
* These steps together ensure that later steps of the process don't need to keep track of whether the queries
* are present in codeql/java-queries or in our own query pack. They just need to resolve the query.
*
* @param cliServer The CodeQL CLI server to use.
* @param queryDir The directory to set up.
* @param language The language to use for the queries.
* @returns true if the setup was successful, false otherwise.
@@ -17,34 +35,104 @@ export async function setUpPack(
queryDir: string,
language: QueryLanguage,
): Promise<boolean> {
// Create the external API query
const externalApiQuerySuccess = await prepareExternalApiQuery(
queryDir,
language,
);
if (!externalApiQuerySuccess) {
return false;
}
// Set up a synthetic pack so that the query can be resolved later.
const syntheticQueryPack = {
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${language}-all`]: "*",
},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
await cliServer.packInstall(queryDir);
// Install the other needed query packs
// Download the required query packs
await cliServer.packDownload([`codeql/${language}-queries`]);
// We'll only check if the application mode query exists in the pack and assume that if it does,
// the framework mode query will also exist.
const applicationModeQuery = await resolveEndpointsQuery(
cliServer,
language,
Mode.Application,
[],
[],
);
if (applicationModeQuery) {
// Set up a synthetic pack so CodeQL doesn't crash later when we try
// to resolve a query within this directory
const syntheticQueryPack = {
name: syntheticQueryPackName,
version: "0.0.0",
dependencies: {},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
} else {
// If we can't resolve the query, we need to write them to desk ourselves.
const externalApiQuerySuccess = await prepareExternalApiQuery(
queryDir,
language,
);
if (!externalApiQuerySuccess) {
return false;
}
// Set up a synthetic pack so that the query can be resolved later.
const syntheticQueryPack = {
name: syntheticQueryPackName,
version: "0.0.0",
dependencies: {
[`codeql/${language}-all`]: "*",
},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
await cliServer.packInstall(queryDir);
}
// Download any other required packs
if (language === "java" && showLlmGeneration()) {
await cliServer.packDownload([`codeql/${language}-automodel-queries`]);
}
return true;
}
/**
* Resolve the query path to the model editor endpoints query. All queries are tagged like this:
* modeleditor endpoints <mode>
* Example: modeleditor endpoints framework-mode
*
* @param cliServer The CodeQL CLI server to use.
* @param language The language of the query pack to use.
* @param mode The mode to resolve the query for.
* @param additionalPackNames Additional pack names to search.
* @param additionalPackPaths Additional pack paths to search.
*/
export async function resolveEndpointsQuery(
cliServer: CodeQLCliServer,
language: string,
mode: Mode,
additionalPackNames: string[] = [],
additionalPackPaths: string[] = [],
): Promise<string | undefined> {
const packsToSearch = [`codeql/${language}-queries`, ...additionalPackNames];
// First, resolve the query that we want to run.
// All queries are tagged like this:
// internal extract automodel <mode> <queryTag>
// Example: internal extract automodel framework-mode candidates
const queries = await resolveQueriesFromPacks(
cliServer,
packsToSearch,
{
kind: "table",
"tags contain all": ["modeleditor", "endpoints", modeTag(mode)],
},
additionalPackPaths,
);
if (queries.length > 1) {
throw new Error(
`Found multiple endpoints queries for ${mode}. Can't continue`,
);
}
if (queries.length === 0) {
return undefined;
}
return queries[0];
}

21
extensions/ql-vscode/src/model-editor/modeling-store.ts
View File

@@ -264,6 +264,27 @@ export class ModelingStore extends DisposableObject {
});
}
public getSelectedMethodDetails() {
const dbState = this.getStateForActiveDb();
if (!dbState) {
throw new Error("No active state found in modeling store");
}
const selectedMethod = dbState.selectedMethod;
if (!selectedMethod) {
return undefined;
}
return {
method: selectedMethod,
usage: dbState.selectedUsage,
modeledMethod: dbState.modeledMethods[selectedMethod.signature],
isModified: dbState.modifiedMethodSignatures.has(
selectedMethod.signature,
),
};
}
private getState(databaseItem: DatabaseItem): DbModelingState {
if (!this.state.has(databaseItem.databaseUri.toString())) {
throw Error(

84
extensions/ql-vscode/test/unit-tests/common/split-buffer.test.ts Normal file
View File

@@ -0,0 +1,84 @@
import { LINE_ENDINGS, SplitBuffer } from "../../../src/common/split-stream";
interface Chunk {
chunk: string;
lines: string[];
}
function checkLines(
buffer: SplitBuffer,
expectedLinesForChunk: string[],
chunkIndex: number | "end",
): void {
expectedLinesForChunk.forEach((expectedLine, lineIndex) => {
const line = buffer.getNextLine();
const location = `[chunk ${chunkIndex}, line ${lineIndex}]: `;
expect(location + line).toEqual(location + expectedLine);
});
expect(buffer.getNextLine()).toBeUndefined();
}
function testSplitBuffer(chunks: Chunk[], endLines: string[]): void {
const buffer = new SplitBuffer(LINE_ENDINGS);
chunks.forEach((chunk, chunkIndex) => {
buffer.addChunk(Buffer.from(chunk.chunk, "utf-8"));
checkLines(buffer, chunk.lines, chunkIndex);
});
buffer.end();
checkLines(buffer, endLines, "end");
}
describe("split buffer", () => {
it("should handle a one-chunk string with no terminator", async () => {
// Won't return the line until we call `end()`.
testSplitBuffer([{ chunk: "some text", lines: [] }], ["some text"]);
});
it("should handle a one-chunk string with a one-byte terminator", async () => {
// Won't return the line until we call `end()` because the actual terminator is shorter than the
// longest terminator.
testSplitBuffer([{ chunk: "some text\n", lines: [] }], ["some text"]);
});
it("should handle a one-chunk string with a two-byte terminator", async () => {
testSplitBuffer([{ chunk: "some text\r\n", lines: ["some text"] }], []);
});
it("should handle a multi-chunk string with terminators at the end of each chunk", async () => {
testSplitBuffer(
[
{ chunk: "first line\n", lines: [] }, // Waiting for second potential terminator byte
{ chunk: "second line\r", lines: ["first line"] }, // Waiting for second potential terminator byte
{ chunk: "third line\r\n", lines: ["second line", "third line"] }, // No wait, because we're at the end
],
[],
);
});
it("should handle a multi-chunk string with terminators at random offsets", async () => {
testSplitBuffer(
[
{ chunk: "first line\nsecond", lines: ["first line"] },
{
chunk: " line\rthird line",
lines: ["second line"],
},
{ chunk: "\r\n", lines: ["third line"] },
],
[],
);
});
it("should handle a terminator split between chunks", async () => {
testSplitBuffer(
[
{ chunk: "first line\r", lines: [] },
{
chunk: "\nsecond line",
lines: ["first line"],
},
],
["second line"],
);
});
});

31
extensions/ql-vscode/test/vscode-tests/activated-extension/databases/db-panel.test.ts
View File

@@ -21,7 +21,13 @@ import { AllCommands } from "../../../../src/common/commands";
jest.setTimeout(60_000);
describe("Db panel UI commands", () => {
// This test has some serious problems:
// - all tests use the same dbConfig file, hence the tests depend on ORDER and have to use the same list name!
// - we depend on highlighted list items when adding a repo to a list. If there's not enough time in between, a test might think a list is highlighted that doesn't exist anymore
let storagePath: string;
let dbConfigFilePath: string;
const commandManager = createVSCodeCommandManager<AllCommands>();
beforeEach(async () => {
@@ -29,6 +35,11 @@ describe("Db panel UI commands", () => {
storagePath =
extension.ctx.storageUri?.fsPath || extension.ctx.globalStorageUri.fsPath;
dbConfigFilePath = path.join(
storagePath,
DbConfigStore.databaseConfigFileName,
);
});
it("should add new remote db list", async () => {
@@ -39,10 +50,6 @@ describe("Db panel UI commands", () => {
);
// Check db config
const dbConfigFilePath = path.join(
storagePath,
DbConfigStore.databaseConfigFileName,
);
const dbConfig: DbConfig = await readJson(dbConfigFilePath);
expect(dbConfig.databases.variantAnalysis.repositoryLists).toHaveLength(1);
expect(dbConfig.databases.variantAnalysis.repositoryLists[0].name).toBe(
@@ -61,10 +68,6 @@ describe("Db panel UI commands", () => {
);
// Check db config
const dbConfigFilePath = path.join(
storagePath,
DbConfigStore.databaseConfigFileName,
);
const dbConfig: DbConfig = await readJson(dbConfigFilePath);
expect(dbConfig.databases.local.lists).toHaveLength(1);
expect(dbConfig.databases.local.lists[0].name).toBe("my-list-1");
@@ -82,10 +85,6 @@ describe("Db panel UI commands", () => {
);
// Check db config
const dbConfigFilePath = path.join(
storagePath,
DbConfigStore.databaseConfigFileName,
);
const dbConfig: DbConfig = await readJson(dbConfigFilePath);
expect(dbConfig.databases.variantAnalysis.repositories).toHaveLength(1);
expect(dbConfig.databases.variantAnalysis.repositories[0]).toBe(
@@ -105,10 +104,6 @@ describe("Db panel UI commands", () => {
);
// Check db config
const dbConfigFilePath = path.join(
storagePath,
DbConfigStore.databaseConfigFileName,
);
const dbConfig: DbConfig = await readJson(dbConfigFilePath);
expect(dbConfig.databases.variantAnalysis.owners).toHaveLength(1);
expect(dbConfig.databases.variantAnalysis.owners[0]).toBe("owner1");
@@ -128,10 +123,6 @@ describe("Db panel UI commands", () => {
);
// Check db config
const dbConfigFilePath = path.join(
storagePath,
DbConfigStore.databaseConfigFileName,
);
const dbConfig: DbConfig = await readJson(dbConfigFilePath);
expect(dbConfig.selected).toBeDefined();
expect(dbConfig.selected).toEqual({

6
extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/external-api-usage-query.test.ts
View File

@@ -46,6 +46,9 @@ describe("external api usage query", () => {
resolveQlpacks: jest.fn().mockResolvedValue({
"my/extensions": "/a/b/c/",
}),
resolveQueriesInSuite: jest
.fn()
.mockResolvedValue(["/a/b/c/ApplicationModeEndpoints.ql"]),
packPacklist: jest
.fn()
.mockResolvedValue([
@@ -104,6 +107,9 @@ describe("external api usage query", () => {
resolveQlpacks: jest.fn().mockResolvedValue({
"my/extensions": "/a/b/c/",
}),
resolveQueriesInSuite: jest
.fn()
.mockResolvedValue(["/a/b/c/ApplicationModeEndpoints.ql"]),
packPacklist: jest
.fn()
.mockResolvedValue([

51
extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/model-editor-queries.test.ts
View File

@@ -10,24 +10,29 @@ import { mockedObject } from "../../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
describe("setUpPack", () => {
const cliServer = mockedObject<CodeQLCliServer>({
packDownload: jest.fn(),
packInstall: jest.fn(),
let queryDir: string;
beforeEach(async () => {
queryDir = dirSync({ unsafeCleanup: true }).name;
});
const languages = Object.keys(fetchExternalApiQueries).flatMap((lang) => {
const queryDir = dirSync({ unsafeCleanup: true }).name;
const query = fetchExternalApiQueries[lang as QueryLanguage];
if (!query) {
return [];
}
return { language: lang as QueryLanguage, queryDir, query };
return { language: lang as QueryLanguage, query };
});
test.each(languages)(
"should create files for $language",
async ({ language, queryDir, query }) => {
describe.each(languages)("for language $language", ({ language, query }) => {
test("should create the files when not found", async () => {
const cliServer = mockedObject<CodeQLCliServer>({
packDownload: jest.fn(),
packInstall: jest.fn(),
resolveQueriesInSuite: jest.fn().mockResolvedValue([]),
});
await setUpPack(cliServer, queryDir, language);
const queryFiles = await readdir(queryDir);
@@ -74,6 +79,32 @@ describe("setUpPack", () => {
contents,
);
}
},
);
});
test("should not create the files when found", async () => {
const cliServer = mockedObject<CodeQLCliServer>({
packDownload: jest.fn(),
packInstall: jest.fn(),
resolveQueriesInSuite: jest
.fn()
.mockResolvedValue(["/a/b/c/ApplicationModeEndpoints.ql"]),
});
await setUpPack(cliServer, queryDir, language);
const queryFiles = await readdir(queryDir);
expect(queryFiles.sort()).toEqual(["codeql-pack.yml"].sort());
const suiteFileContents = await readFile(
join(queryDir, "codeql-pack.yml"),
"utf8",
);
const suiteYaml = load(suiteFileContents);
expect(suiteYaml).toEqual({
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {},
});
});
});
});