mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2065 from github/aeisenberg/run-with-all-data-extensions

Browse Source 
Add ability to run query with data extensions
This commit is contained in:
Andrew Eisenberg
2023-03-10 08:09:19 -08:00
committed by GitHub
parent e3d8dbc484 790a152f42
commit 0d8df9ad88
13 changed files with 201 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.vscode/settings.json vendored
View File

@@ -42,22 +42,29 @@
"LANG": "en-US",
"TZ": "UTC"
},
// These options are used by the `jestrunner.debug` command.
// They are not used by the `jestrunner.run` command.
// After clicking "debug" over a test, continually invoke the
// "Debug: Attach to Node Process" command until you see a
// process named "Code Helper (Plugin)". Then click "attach".
// This will attach the debugger to the test process.
"jestrunner.debugOptions": {
// Uncomment to debug integration tests
// "attachSimplePort": 9223,
"attachSimplePort": 9223,
"env": {
"LANG": "en-US",
"TZ": "UTC",
// Uncomment to set a custom path to a CodeQL checkout.
// "TEST_CODEQL_PATH": "../codeql",
// "TEST_CODEQL_PATH": "/absolute/path/to/checkout/of/codeql",
// Uncomment to set a custom path to a CodeQL CLI executable.
// This is the CodeQL version that will be used in the tests.
// "CLI_PATH": "/path/to/customg/codeql",
// "CLI_PATH": "/absolute/path/to/custom/codeql",
// Uncomment to debug integration tests
// "VSCODE_WAIT_FOR_DEBUGGER": "true",
"VSCODE_WAIT_FOR_DEBUGGER": "true",
}
},
"terminal.integrated.env.linux": {

13
extensions/ql-vscode/package.json
View File

@@ -224,6 +224,19 @@
"default": true,
"description": "Enable the 'Quick Evaluation' CodeLens."
},
"codeQL.runningQueries.useExtensionPacks": {
"type": "string",
"default": "none",
"enum": [
"none",
"all"
],
"enumDescriptions": [
"Do not use extension packs.",
"Use all extension packs found in the workspace."
],
"description": "Choose whether or not to run queries using extension packs. Requires CodeQL CLI v2.12.3 or later."
},
"codeQL.resultsDisplay.pageSize": {
"type": "integer",
"default": 200,

46
extensions/ql-vscode/src/cli.ts
View File

@@ -1163,24 +1163,32 @@ export class CodeQLCliServer implements Disposable {
/**
* Gets information about available qlpacks
* @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
* @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
* the default CLI search path is used.
* @param additionalPacks A list of directories to search for qlpacks.
* @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
* be returned. If false, all packs will be returned.
* @returns A dictionary mapping qlpack name to the directory it comes from
*/
resolveQlpacks(
async resolveQlpacks(
additionalPacks: string[],
searchPath?: string[],
extensionPacksOnly = false,
): Promise<QlpacksInfo> {
const args = this.getAdditionalPacksArg(additionalPacks);
if (searchPath?.length) {
args.push("--search-path", join(...searchPath));
if (extensionPacksOnly) {
if (!(await this.cliConstraints.supportsQlpacksKind())) {
void this.logger.log(
"Warning: Running with extension packs is only supported by CodeQL CLI v2.12.3 or later.",
);
return {};
}
args.push("--kind", "extension", "--no-recursive");
}
return this.runJsonCodeQlCliCommand<QlpacksInfo>(
["resolve", "qlpacks"],
args,
"Resolving qlpack information",
`Resolving qlpack information${
extensionPacksOnly ? " (extension packs only)" : ""
}`,
);
}
@@ -1380,6 +1388,17 @@ export class CodeQLCliServer implements Disposable {
private getAdditionalPacksArg(paths: string[]): string[] {
return paths.length ? ["--additional-packs", paths.join(delimiter)] : [];
}
public async useExtensionPacks(): Promise<boolean> {
return (
this.cliConfig.useExtensionPacks &&
(await this.cliConstraints.supportsQlpacksKind())
);
}
public async setUseExtensionPacks(useExtensionPacks: boolean) {
await this.cliConfig.setUseExtensionPacks(useExtensionPacks);
}
}
/**
@@ -1668,6 +1687,11 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_WORKSPACE_RFERENCES = new SemVer("2.11.3");
/**
* CLI version that supports the `--kind` option for the `resolve qlpacks` command.
*/
public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
constructor(private readonly cli: CodeQLCliServer) {
/**/
}
@@ -1725,4 +1749,10 @@ export class CliVersionConstraint {
CliVersionConstraint.CLI_VERSION_WITH_WORKSPACE_RFERENCES,
);
}
async supportsQlpacksKind() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
);
}
}

20
extensions/ql-vscode/src/config.ts
View File

@@ -137,6 +137,10 @@ const DEBUG_SETTING = new Setting("debug", RUNNING_QUERIES_SETTING);
const MAX_PATHS = new Setting("maxPaths", RUNNING_QUERIES_SETTING);
const RUNNING_TESTS_SETTING = new Setting("runningTests", ROOT_SETTING);
const RESULTS_DISPLAY_SETTING = new Setting("resultsDisplay", ROOT_SETTING);
const USE_EXTENSION_PACKS = new Setting(
"useExtensionPacks",
RUNNING_QUERIES_SETTING,
);
export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting(
"additionalTestArguments",
@@ -196,6 +200,7 @@ const CLI_SETTINGS = [
NUMBER_OF_TEST_THREADS_SETTING,
NUMBER_OF_THREADS_SETTING,
MAX_PATHS,
USE_EXTENSION_PACKS,
];
export interface CliConfig {
@@ -203,7 +208,9 @@ export interface CliConfig {
numberTestThreads: number;
numberThreads: number;
maxPaths: number;
useExtensionPacks: boolean;
onDidChangeConfiguration?: Event<void>;
setUseExtensionPacks: (useExtensionPacks: boolean) => Promise<void>;
}
export abstract class ConfigListener extends DisposableObject {
@@ -400,6 +407,19 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
return MAX_PATHS.getValue<number>();
}
public get useExtensionPacks(): boolean {
// currently, we are restricting the values of this setting to 'all' or 'none'.
return USE_EXTENSION_PACKS.getValue() === "all";
}
// Exposed for testing only
public async setUseExtensionPacks(newUseExtensionPacks: boolean) {
await USE_EXTENSION_PACKS.updateValue(
newUseExtensionPacks ? "all" : "none",
ConfigurationTarget.Global,
);
}
protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
}

1
extensions/ql-vscode/src/pure/new-messages.ts
View File

@@ -126,6 +126,7 @@ export interface RunQueryParams {
singletonExternalInputs: Record<string, string>;
dilPath?: string;
logPath?: string;
extensionPacks?: string[];
}
export interface RunQueryResult {

5
extensions/ql-vscode/src/query-server/run-queries.ts
View File

@@ -70,6 +70,10 @@ export async function compileAndRunQueryAgainstDatabase(
: { query: {} };
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
const extensionPacks = (await qs.cliServer.useExtensionPacks())
? Object.keys(await qs.cliServer.resolveQlpacks(diskWorkspaceFolders, true))
: undefined;
const db = dbItem.databaseUri.fsPath;
const logPath = queryInfo ? query.evalLogPath : undefined;
const queryToRun: messages.RunQueryParams = {
@@ -82,6 +86,7 @@ export async function compileAndRunQueryAgainstDatabase(
dilPath: query.dilPath,
logPath,
target,
extensionPacks,
};
await query.createTimestampFile();
let result: messages.RunQueryResult | undefined;

8
extensions/ql-vscode/test/data-extensions/extension-pack/ext/extension.yml Normal file
View File

@@ -0,0 +1,8 @@
extensions:
- data:
- [2]
- [3]
- [4]
addsTo:
extensible: testExtensible
pack: semmle/has-extension

8
extensions/ql-vscode/test/data-extensions/extension-pack/qlpack.yml Normal file
View File

@@ -0,0 +1,8 @@
name: semmle/targets-extension
library: true
version: 0.0.0
extensionTargets:
semmle/has-extension: '*'
dataExtensions:
- ext/*

7
extensions/ql-vscode/test/data-extensions/pack-using-extensions/ext/extension.yml Normal file
View File

@@ -0,0 +1,7 @@
extensions:
- data:
- [1]
addsTo:
extensible: testExtensible
pack: semmle/has-extension

6
extensions/ql-vscode/test/data-extensions/pack-using-extensions/qlpack.yml Normal file
View File

@@ -0,0 +1,6 @@
name: semmle/has-extension
version: 0.0.0
dependencies:
codeql/javascript-all: '*'
dataExtensions:
- ext/*

8
extensions/ql-vscode/test/data-extensions/pack-using-extensions/query.ql Normal file
View File

@@ -0,0 +1,8 @@
import javascript
extensible predicate testExtensible(int i);
from int i
where testExtensible(i)
select i

1
extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-vscode.config.js
View File

@@ -18,6 +18,7 @@ const config = {
"--disable-extension",
"github.copilot",
path.resolve(rootDir, "test/data"),
path.resolve(rootDir, "test/data-extensions"), // folder containing the extension packs and packs that are targeted by the extension pack
// CLI integration tests requires a multi-root workspace so that the data and the QL sources are accessible.
...(process.env.TEST_CODEQL_PATH ? [process.env.TEST_CODEQL_PATH] : []),
],

76
extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
View File

@@ -19,11 +19,13 @@ import { DatabaseItem, DatabaseManager } from "../../../src/local-databases";
import { CodeQLExtensionInterface } from "../../../src/extension";
import { cleanDatabases, dbLoc, storagePath } from "../global.helper";
import { importArchiveDatabase } from "../../../src/databaseFetcher";
import { CodeQLCliServer } from "../../../src/cli";
import { CliVersionConstraint, CodeQLCliServer } from "../../../src/cli";
import { describeWithCodeQL } from "../cli";
import { tmpDir } from "../../../src/helpers";
import { createInitialQueryInfo } from "../../../src/run-queries-shared";
import { QueryRunner } from "../../../src/queryRunner";
import { CompletedQueryInfo } from "../../../src/query-results";
import { SELECT_QUERY_NAME } from "../../../src/contextual/locationFinder";
jest.setTimeout(20_000);
@@ -96,6 +98,78 @@ describeWithCodeQL()("Queries", () => {
await cleanDatabases(databaseManager);
});
describe("extension packs", () => {
const queryUsingExtensionPath = join(
__dirname,
"../..",
"data-extensions",
"pack-using-extensions",
"query.ql",
);
it("should run a query that has an extension without looking for extensions in the workspace", async () => {
if (!(await supportsExtensionPacks())) {
console.log(
`Skipping test because it is only supported for CodeQL CLI versions >= ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND}`,
);
return;
}
await cli.setUseExtensionPacks(false);
const parsedResults = await runQueryWithExtensions();
expect(parsedResults).toEqual([1]);
});
it("should run a query that has an extension and look for extensions in the workspace", async () => {
if (!(await supportsExtensionPacks())) {
return;
}
await cli.setUseExtensionPacks(true);
const parsedResults = await runQueryWithExtensions();
expect(parsedResults).toEqual([1, 2, 3, 4]);
});
async function supportsExtensionPacks(): Promise<boolean> {
if (await qs.cliServer.cliConstraints.supportsQlpacksKind()) {
return true;
}
console.log(
`Skipping test because it is only supported for CodeQL CLI versions >= ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND}`,
);
return false;
}
async function runQueryWithExtensions() {
const result = new CompletedQueryInfo(
await qs.compileAndRunQueryAgainstDatabase(
dbItem,
await mockInitialQueryInfo(queryUsingExtensionPath),
join(tmpDir.name, "mock-storage-path"),
progress,
token,
),
);
// Check that query was successful
expect(result.successful).toBe(true);
// Load query results
const chunk = await qs.cliServer.bqrsDecode(
result.getResultsPath(SELECT_QUERY_NAME, true),
SELECT_QUERY_NAME,
{
// there should only be one result
offset: 0,
pageSize: 10,
},
);
// Extract the results as an array.
return chunk.tuples.map((t) => t[0]);
}
});
it("should run a query", async () => {
const queryPath = join(__dirname, "data", "simple-query.ql");
const result = qs.compileAndRunQueryAgainstDatabase(