mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3163 from github/koesie10/queries-panel-no-cli-server

Browse Source 
Do not use the CLI server to determine query pack language
This commit is contained in:
Koen Vlaswinkel
2024-01-03 09:36:20 +01:00
committed by GitHub
parent 870874d7f3 ed4b296e41
commit 1ad20fa354
15 changed files with 418 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
extensions/ql-vscode/scripts/generate-schemas.ts
View File

@@ -6,6 +6,16 @@ import { format, resolveConfig } from "prettier";
const extensionDirectory = resolve(__dirname, "..");
const schemas = [
{
path: join(extensionDirectory, "src", "packaging", "qlpack-file.ts"),
type: "QlPackFile",
schemaPath: join(
extensionDirectory,
"src",
"packaging",
"qlpack-file.schema.json",
),
},
{
path: join(
extensionDirectory,

23
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -738,29 +738,6 @@ export class CodeQLCliServer implements Disposable {
);
}
/**
* Resolve the library path and dbscheme for a query.
* @param workspaces The current open workspaces
* @param queryPath The path to the query
*/
async resolveLibraryPath(
workspaces: string[],
queryPath: string,
silent = false,
): Promise<QuerySetup> {
const subcommandArgs = [
"--query",
queryPath,
...this.getAdditionalPacksArg(workspaces),
];
return await this.runJsonCodeQlCliCommand<QuerySetup>(
["resolve", "library-path"],
subcommandArgs,
"Resolving library paths",
{ silent },
);
}
/**
* Resolves the language for a query.
* @param queryUri The URI of the query

4
extensions/ql-vscode/src/common/discovery.ts
View File

@@ -1,6 +1,6 @@
import { DisposableObject } from "./disposable-object";
import { getErrorMessage } from "./helpers-pure";
import { Logger } from "./logging";
import { BaseLogger } from "./logging";
/**
* Base class for "discovery" operations, which scan the file system to find specific kinds of
@@ -13,7 +13,7 @@ export abstract class Discovery extends DisposableObject {
constructor(
protected readonly name: string,
private readonly logger: Logger,
protected readonly logger: BaseLogger,
) {
super();
}

26
extensions/ql-vscode/src/common/qlpack-language.ts Normal file
View File

@@ -0,0 +1,26 @@
import { QueryLanguage } from "./query-language";
import { loadQlpackFile } from "../packaging/qlpack-file-loader";
/**
* @param qlpackPath The path to the `qlpack.yml` or `codeql-pack.yml` file.
* @return the language of the given qlpack file, or undefined if the file is
* not a valid qlpack file or does not contain exactly one language.
*/
export async function getQlPackLanguage(
qlpackPath: string,
): Promise<QueryLanguage | undefined> {
const qlPack = await loadQlpackFile(qlpackPath);
const dependencies = qlPack?.dependencies;
if (!dependencies) {
return;
}
const matchingLanguages = Object.values(QueryLanguage).filter(
(language) => `codeql/${language}-all` in dependencies,
);
if (matchingLanguages.length !== 1) {
return undefined;
}
return matchingLanguages[0];
}

6
extensions/ql-vscode/src/extension.ts
View File

@@ -800,11 +800,7 @@ async function activateWithInstalledDistribution(
);
ctx.subscriptions.push(databaseUI);
const queriesModule = QueriesModule.initialize(
app,
languageContext,
cliServer,
);
const queriesModule = QueriesModule.initialize(app, languageContext);
void extLogger.log("Initializing evaluator log viewer.");
const evalLogViewer = new EvalLogViewer();

25
extensions/ql-vscode/src/local-queries/skeleton-query-wizard.ts
View File

@@ -28,7 +28,7 @@ import {
isCodespacesTemplate,
setQlPackLocation,
} from "../config";
import { lstat, pathExists, readFile } from "fs-extra";
import { lstat, pathExists } from "fs-extra";
import { askForLanguage } from "../codeql-cli/query-language";
import { showInformationMessageWithAction } from "../common/vscode/dialog";
import { redactableError } from "../common/errors";
@@ -36,8 +36,7 @@ import { App } from "../common/app";
import { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
import { containsPath, pathsEqual } from "../common/files";
import { getQlPackPath } from "../common/ql";
import { load } from "js-yaml";
import { QlPackFile } from "../packaging/qlpack-file";
import { getQlPackLanguage } from "../common/qlpack-language";
type QueryLanguagesToDatabaseMap = Record<string, string>;
@@ -253,24 +252,12 @@ export class SkeletonQueryWizard {
return undefined;
}
const qlPack = load(await readFile(qlPackPath, "utf8")) as
| QlPackFile
| undefined;
const dependencies = qlPack?.dependencies;
if (!dependencies || typeof dependencies !== "object") {
return;
const language = await getQlPackLanguage(qlPackPath);
if (language) {
this.qlPackStoragePath = matchingQueryPackPath;
}
const matchingLanguages = Object.values(QueryLanguage).filter(
(language) => `codeql/${language}-all` in dependencies,
);
if (matchingLanguages.length !== 1) {
return undefined;
}
this.qlPackStoragePath = matchingQueryPackPath;
return matchingLanguages[0];
return language;
}
private async chooseLanguage() {

66
extensions/ql-vscode/src/model-editor/extension-pack-metadata.schema.json
View File

@@ -5,11 +5,24 @@
"ExtensionPackMetadata": {
"type": "object",
"properties": {
"name": {
"type": ["string", "null"]
},
"version": {
"type": ["string", "null"]
},
"extensionTargets": {
"type": "object",
"additionalProperties": {
"type": "string"
}
"anyOf": [
{
"type": "object",
"additionalProperties": {
"type": "string"
}
},
{
"type": "null"
}
]
},
"dataExtensions": {
"anyOf": [
@@ -21,35 +34,46 @@
},
{
"type": "string"
},
{
"type": "null"
}
]
},
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"dependencies": {
"type": "object",
"additionalProperties": {
"type": "string"
}
"anyOf": [
{
"type": "object",
"additionalProperties": {
"type": "string"
}
},
{
"type": "null"
}
]
},
"dbscheme": {
"type": "string"
"type": ["string", "null"]
},
"library": {
"type": "boolean"
"type": ["boolean", "null"]
},
"defaultSuite": {
"type": "array",
"items": {
"$ref": "#/definitions/SuiteInstruction"
}
"anyOf": [
{
"type": "array",
"items": {
"$ref": "#/definitions/SuiteInstruction"
}
},
{
"type": "null"
}
]
},
"defaultSuiteFile": {
"type": "string"
"type": ["string", "null"]
}
},
"required": ["dataExtensions", "extensionTargets", "name", "version"]

4
extensions/ql-vscode/src/model-editor/extension-pack-metadata.ts
View File

@@ -1,7 +1,9 @@
import { QlPackFile } from "../packaging/qlpack-file";
export type ExtensionPackMetadata = QlPackFile & {
// Make both extensionTargets and dataExtensions required
// Make name, version, extensionTargets, and dataExtensions required
name: string;
version: string;
extensionTargets: Record<string, string>;
dataExtensions: string[] | string;
};

32
extensions/ql-vscode/src/packaging/qlpack-file-loader.ts Normal file
View File

@@ -0,0 +1,32 @@
import Ajv from "ajv";
import * as qlpackFileSchemaJson from "./qlpack-file.schema.json";
import { QlPackFile } from "./qlpack-file";
import { load } from "js-yaml";
import { readFile } from "fs-extra";
const ajv = new Ajv({ allErrors: true });
const qlpackFileValidate = ajv.compile(qlpackFileSchemaJson);
export async function loadQlpackFile(path: string): Promise<QlPackFile> {
const qlpackFileText = await readFile(path, "utf8");
let qlPack = load(qlpackFileText) as QlPackFile | undefined;
if (qlPack === undefined || qlPack === null) {
// An empty file is not valid according to the schema since it's not an object,
// but it is equivalent to an empty object.
qlPack = {};
}
qlpackFileValidate(qlPack);
if (qlpackFileValidate.errors) {
throw new Error(
`Invalid extension pack YAML: ${qlpackFileValidate.errors
.map((error) => `${error.instancePath} ${error.message}`)
.join(", ")}`,
);
}
return qlPack;
}

121
extensions/ql-vscode/src/packaging/qlpack-file.schema.json Normal file
View File

@@ -0,0 +1,121 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/QlPackFile",
"definitions": {
"QlPackFile": {
"type": "object",
"properties": {
"name": {
"type": ["string", "null"]
},
"version": {
"type": ["string", "null"]
},
"dependencies": {
"anyOf": [
{
"type": "object",
"additionalProperties": {
"type": "string"
}
},
{
"type": "null"
}
]
},
"extensionTargets": {
"anyOf": [
{
"type": "object",
"additionalProperties": {
"type": "string"
}
},
{
"type": "null"
}
]
},
"dbscheme": {
"type": ["string", "null"]
},
"library": {
"type": ["boolean", "null"]
},
"defaultSuite": {
"anyOf": [
{
"type": "array",
"items": {
"$ref": "#/definitions/SuiteInstruction"
}
},
{
"type": "null"
}
]
},
"defaultSuiteFile": {
"type": ["string", "null"]
},
"dataExtensions": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
},
{
"type": "null"
}
]
}
},
"description": "The qlpack pack file, either in qlpack.yml or in codeql-pack.yml."
},
"SuiteInstruction": {
"type": "object",
"properties": {
"qlpack": {
"type": "string"
},
"query": {
"type": "string"
},
"queries": {
"type": "string"
},
"include": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"exclude": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"from": {
"type": "string"
}
},
"description": "A single entry in a .qls file."
}
}
}

18
extensions/ql-vscode/src/packaging/qlpack-file.ts
View File

@@ -4,13 +4,13 @@ import { SuiteInstruction } from "./suite-instruction";
* The qlpack pack file, either in qlpack.yml or in codeql-pack.yml.
*/
export interface QlPackFile {
name: string;
version: string;
dependencies?: Record<string, string>;
extensionTargets?: Record<string, string>;
dbscheme?: string;
library?: boolean;
defaultSuite?: SuiteInstruction[];
defaultSuiteFile?: string;
dataExtensions?: string[] | string;
name?: string | null;
version?: string | null;
dependencies?: Record<string, string> | null;
extensionTargets?: Record<string, string> | null;
dbscheme?: string | null;
library?: boolean | null;
defaultSuite?: SuiteInstruction[] | null;
defaultSuiteFile?: string | null;
dataExtensions?: string[] | string | null;
}

12
extensions/ql-vscode/src/queries-panel/queries-module.ts
View File

@@ -1,4 +1,3 @@
import { CodeQLCliServer } from "../codeql-cli/cli";
import { extLogger } from "../common/logging/vscode";
import { App } from "../common/app";
import { DisposableObject } from "../common/disposable-object";
@@ -26,23 +25,18 @@ export class QueriesModule extends DisposableObject {
public static initialize(
app: App,
languageContext: LanguageContextStore,
cliServer: CodeQLCliServer,
): QueriesModule {
const queriesModule = new QueriesModule(app);
app.subscriptions.push(queriesModule);
queriesModule.initialize(app, languageContext, cliServer);
queriesModule.initialize(app, languageContext);
return queriesModule;
}
private initialize(
app: App,
langauageContext: LanguageContextStore,
cliServer: CodeQLCliServer,
): void {
private initialize(app: App, langauageContext: LanguageContextStore): void {
void extLogger.log("Initializing queries panel.");
const queryPackDiscovery = new QueryPackDiscovery(cliServer);
const queryPackDiscovery = new QueryPackDiscovery();
this.push(queryPackDiscovery);
void queryPackDiscovery.initialRefresh();

37
extensions/ql-vscode/src/queries-panel/query-pack-discovery.ts
View File

@@ -1,14 +1,11 @@
import { basename, dirname } from "path";
import { CodeQLCliServer, QuerySetup } from "../codeql-cli/cli";
import { Event } from "vscode";
import { QueryLanguage, dbSchemeToLanguage } from "../common/query-language";
import { QueryLanguage } from "../common/query-language";
import { FALLBACK_QLPACK_FILENAME, QLPACK_FILENAMES } from "../common/ql";
import { FilePathDiscovery } from "../common/vscode/file-path-discovery";
import { getErrorMessage } from "../common/helpers-pure";
import { extLogger } from "../common/logging/vscode";
import { EOL } from "os";
import { containsPath } from "../common/files";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { getQlPackLanguage } from "../common/qlpack-language";
import { getErrorMessage } from "../common/helpers-pure";
interface QueryPack {
path: string;
@@ -19,7 +16,7 @@ interface QueryPack {
* Discovers all query packs in the workspace.
*/
export class QueryPackDiscovery extends FilePathDiscovery<QueryPack> {
constructor(private readonly cliServer: CodeQLCliServer) {
constructor() {
super("Query Pack Discovery", `**/{${QLPACK_FILENAMES.join(",")}}`);
}
@@ -71,32 +68,18 @@ export class QueryPackDiscovery extends FilePathDiscovery<QueryPack> {
}
protected async getDataForPath(path: string): Promise<QueryPack> {
const language = await this.determinePackLanguage(path);
return { path, language };
}
private async determinePackLanguage(
path: string,
): Promise<QueryLanguage | undefined> {
let packInfo: QuerySetup | undefined = undefined;
let language: QueryLanguage | undefined;
try {
packInfo = await this.cliServer.resolveLibraryPath(
getOnDiskWorkspaceFolders(),
path,
true,
);
language = await getQlPackLanguage(path);
} catch (err) {
void extLogger.log(
`Query pack discovery failed to determine language for query pack: ${path}${EOL}Reason: ${getErrorMessage(
void this.logger.log(
`Query pack discovery failed to determine language for query pack: ${path}\n\tReason: ${getErrorMessage(
err,
)}`,
);
language = undefined;
}
if (packInfo?.dbscheme === undefined) {
return undefined;
}
const dbscheme = basename(packInfo.dbscheme);
return dbSchemeToLanguage[dbscheme];
return { path, language };
}
protected pathIsRelevant(path: string): boolean {

123
extensions/ql-vscode/test/unit-tests/common/qlpack-language.test.ts Normal file
View File

@@ -0,0 +1,123 @@
import { join } from "path";
import { dirSync } from "tmp-promise";
import { DirResult } from "tmp";
import { outputFile } from "fs-extra";
import { dump } from "js-yaml";
import { QueryLanguage } from "../../../src/common/query-language";
import { getQlPackLanguage } from "../../../src/common/qlpack-language";
describe("getQlPackLanguage", () => {
let tmpDir: DirResult;
let qlpackPath: string;
beforeEach(() => {
tmpDir = dirSync({
prefix: "queries_",
keep: false,
unsafeCleanup: true,
});
qlpackPath = join(tmpDir.name, "qlpack.yml");
});
afterEach(() => {
tmpDir.removeCallback();
});
it.each(Object.values(QueryLanguage))(
"should find a single language %s",
async (language) => {
await writeYAML(qlpackPath, {
name: "test",
dependencies: {
[`codeql/${language}-all`]: "^0.7.0",
"my-custom-pack/test": "${workspace}",
},
});
const result = await getQlPackLanguage(qlpackPath);
expect(result).toEqual(language);
},
);
it("should find nothing when there is no dependencies key", async () => {
await writeYAML(qlpackPath, {
name: "test",
});
const result = await getQlPackLanguage(qlpackPath);
expect(result).toEqual(undefined);
});
it("should find nothing when the dependencies are empty", async () => {
await writeYAML(qlpackPath, {
name: "test",
dependencies: {},
});
const result = await getQlPackLanguage(qlpackPath);
expect(result).toEqual(undefined);
});
it("should throw when dependencies is a scalar", async () => {
await writeYAML(qlpackPath, {
name: "test",
dependencies: "codeql/java-all",
});
await expect(getQlPackLanguage(qlpackPath)).rejects.toBeDefined();
});
it("should throw when dependencies is an array", async () => {
await writeYAML(qlpackPath, {
name: "test",
dependencies: ["codeql/java-all"],
});
await expect(getQlPackLanguage(qlpackPath)).rejects.toBeDefined();
});
it("should find nothing when there are no matching dependencies", async () => {
await writeYAML(qlpackPath, {
name: "test",
dependencies: {
"codeql/java-queries": "*",
"github/my-test-query-pack": "*",
},
});
const result = await getQlPackLanguage(qlpackPath);
expect(result).toEqual(undefined);
});
it("should find nothing when there are multiple matching dependencies", async () => {
await writeYAML(qlpackPath, {
name: "test",
dependencies: {
"codeql/java-all": "*",
"codeql/csharp-all": "*",
},
});
const result = await getQlPackLanguage(qlpackPath);
expect(result).toEqual(undefined);
});
it("should throw when the file does not exist", async () => {
await expect(getQlPackLanguage(qlpackPath)).rejects.toBeDefined();
});
it("should throw when reading a directory", async () => {
await expect(getQlPackLanguage(tmpDir.name)).rejects.toBeDefined();
});
it("should throw when the file is invalid YAML", async () => {
await outputFile(qlpackPath, `name: test\n foo: bar`);
await expect(getQlPackLanguage(tmpDir.name)).rejects.toBeDefined();
});
});
async function writeYAML(path: string, yaml: unknown): Promise<void> {
await outputFile(path, dump(yaml), "utf-8");
}

85
extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-pack-discovery.test.ts
View File

@@ -2,9 +2,8 @@ import { Uri, workspace } from "vscode";
import { QueryPackDiscovery } from "../../../../src/queries-panel/query-pack-discovery";
import * as tmp from "tmp";
import { dirname, join } from "path";
import { CodeQLCliServer, QuerySetup } from "../../../../src/codeql-cli/cli";
import { mockedObject } from "../../utils/mocking.helpers";
import { mkdirSync, writeFileSync } from "fs";
import { ensureDir, writeJSON } from "fs-extra";
import { QueryLanguage } from "../../../../src/common/query-language";
describe("Query pack discovery", () => {
let tmpDir: string;
@@ -12,9 +11,6 @@ describe("Query pack discovery", () => {
let workspacePath: string;
let resolveLibraryPath: jest.SpiedFunction<
typeof CodeQLCliServer.prototype.resolveLibraryPath
>;
let discovery: QueryPackDiscovery;
beforeEach(() => {
@@ -34,15 +30,7 @@ describe("Query pack discovery", () => {
.spyOn(workspace, "workspaceFolders", "get")
.mockReturnValue([workspaceFolder]);
const mockResolveLibraryPathValue: QuerySetup = {
libraryPath: [],
dbscheme: "/ql/java/ql/lib/config/semmlecode.dbscheme",
};
resolveLibraryPath = jest
.fn()
.mockResolvedValue(mockResolveLibraryPathValue);
const mockCliServer = mockedObject<CodeQLCliServer>({ resolveLibraryPath });
discovery = new QueryPackDiscovery(mockCliServer);
discovery = new QueryPackDiscovery();
});
afterEach(() => {
@@ -60,7 +48,7 @@ describe("Query pack discovery", () => {
});
it("locates a query pack in the same directory", async () => {
makeTestFile(join(workspacePath, "qlpack.yml"));
await makeTestFile(join(workspacePath, "qlpack.yml"));
await discovery.initialRefresh();
@@ -70,7 +58,7 @@ describe("Query pack discovery", () => {
});
it("locates a query pack using the old pack name", async () => {
makeTestFile(join(workspacePath, "codeql-pack.yml"));
await makeTestFile(join(workspacePath, "codeql-pack.yml"));
await discovery.initialRefresh();
@@ -80,7 +68,7 @@ describe("Query pack discovery", () => {
});
it("locates a query pack in a higher directory", async () => {
makeTestFile(join(workspacePath, "qlpack.yml"));
await makeTestFile(join(workspacePath, "qlpack.yml"));
await discovery.initialRefresh();
@@ -92,7 +80,7 @@ describe("Query pack discovery", () => {
});
it("doesn't recognise a query pack in a sibling directory", async () => {
makeTestFile(join(workspacePath, "foo", "qlpack.yml"));
await makeTestFile(join(workspacePath, "foo", "qlpack.yml"));
await discovery.initialRefresh();
@@ -109,24 +97,11 @@ describe("Query pack discovery", () => {
});
it("query packs override those from parent directories", async () => {
makeTestFile(join(workspacePath, "qlpack.yml"));
makeTestFile(join(workspacePath, "foo", "qlpack.yml"));
resolveLibraryPath.mockImplementation(async (_workspaces, queryPath) => {
if (queryPath === join(workspacePath, "qlpack.yml")) {
return {
libraryPath: [],
dbscheme: "/ql/java/ql/lib/config/semmlecode.dbscheme",
};
}
if (queryPath === join(workspacePath, "foo", "qlpack.yml")) {
return {
libraryPath: [],
dbscheme: "/ql/cpp/ql/lib/semmlecode.cpp.dbscheme",
};
}
throw new Error(`Unknown query pack: ${queryPath}`);
});
await makeTestFile(join(workspacePath, "qlpack.yml"), QueryLanguage.Java);
await makeTestFile(
join(workspacePath, "foo", "qlpack.yml"),
QueryLanguage.Cpp,
);
await discovery.initialRefresh();
@@ -141,24 +116,11 @@ describe("Query pack discovery", () => {
});
it("prefers a query pack called qlpack.yml", async () => {
makeTestFile(join(workspacePath, "qlpack.yml"));
makeTestFile(join(workspacePath, "codeql-pack.yml"));
resolveLibraryPath.mockImplementation(async (_workspaces, queryPath) => {
if (queryPath === join(workspacePath, "qlpack.yml")) {
return {
libraryPath: [],
dbscheme: "/ql/cpp/ql/lib/semmlecode.cpp.dbscheme",
};
}
if (queryPath === join(workspacePath, "codeql-pack.yml")) {
return {
libraryPath: [],
dbscheme: "/ql/java/ql/lib/config/semmlecode.dbscheme",
};
}
throw new Error(`Unknown query pack: ${queryPath}`);
});
await makeTestFile(join(workspacePath, "qlpack.yml"), QueryLanguage.Cpp);
await makeTestFile(
join(workspacePath, "codeql-pack.yml"),
QueryLanguage.Java,
);
await discovery.initialRefresh();
@@ -169,7 +131,14 @@ describe("Query pack discovery", () => {
});
});
function makeTestFile(path: string) {
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, "");
async function makeTestFile(
path: string,
language: QueryLanguage = QueryLanguage.Java,
) {
await ensureDir(dirname(path));
await writeJSON(path, {
dependencies: {
[`codeql/${language}-all`]: "*",
},
});
}