mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Automatically name extension packs

Browse Source 
This will change how extension packs are named in the data extensions
editor. Before, the user had to pick a workspace folder and a name for
the extension pack. Now, the workspace folder will be picked
automatically if we can detect it (i.e. it follows the naming structure
we expect), or the user will still need to select it. The extension pack
name is always auto-generated based on the database name and the
database language.

This adds a new `codeQL.dataExtensions.disableAutoNameExtensionPack`
setting to disable this behavior while we are still working on changing
how the data extensions editor works.
This commit is contained in:
Koen Vlaswinkel
2023-06-16 13:31:31 +02:00
parent 0f9d127b4c
commit 549884d507
3 changed files with 421 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
extensions/ql-vscode/src/config.ts
View File

@@ -714,7 +714,15 @@ export function showQueriesPanel(): boolean {
const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
const DISABLE_AUTO_NAME_EXTENSION_PACK = new Setting(
"disableAutoNameExtensionPack",
DATA_EXTENSIONS,
);
export function showLlmGeneration(): boolean {
return !!LLM_GENERATION.getValue<boolean>();
}
export function disableAutoNameExtensionPack(): boolean {
return !!DISABLE_AUTO_NAME_EXTENSION_PACK.getValue<boolean>();
}

217
extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts
View File

@@ -2,8 +2,8 @@ import { join, relative, resolve, sep } from "path";
import { outputFile, pathExists, readFile } from "fs-extra";
import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import { minimatch } from "minimatch";
import { CancellationToken, window } from "vscode";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { CancellationToken, window, WorkspaceFolder } from "vscode";
import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
import {
getOnDiskWorkspaceFolders,
getOnDiskWorkspaceFoldersObjects,
@@ -15,6 +15,7 @@ import { getErrorMessage } from "../pure/helpers-pure";
import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
import { containsPath } from "../pure/files";
import { disableAutoNameExtensionPack } from "../config";
const maxStep = 3;
@@ -79,6 +80,21 @@ async function pickExtensionPack(
true,
);
if (!disableAutoNameExtensionPack()) {
progress({
message: "Creating extension pack...",
step: 2,
maxStep,
});
return autoCreateExtensionPack(
databaseItem.name,
databaseItem.language,
extensionPacksInfo,
logger,
);
}
if (Object.keys(extensionPacksInfo).length === 0) {
return pickNewExtensionPack(databaseItem, token);
}
@@ -239,26 +255,15 @@ async function pickNewExtensionPack(
databaseItem: Pick<DatabaseItem, "name" | "language">,
token: CancellationToken,
): Promise<ExtensionPack | undefined> {
const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
const workspaceFolderOptions = workspaceFolders.map((folder) => ({
label: folder.name,
detail: folder.uri.fsPath,
path: folder.uri.fsPath,
}));
// We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
// we only want to include on-disk workspace folders.
const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
title: "Select workspace folder to create extension pack in",
});
const workspaceFolder = await askForWorkspaceFolder();
if (!workspaceFolder) {
return undefined;
}
let examplePackName = `${databaseItem.name}-extensions`;
if (!examplePackName.includes("/")) {
examplePackName = `pack/${examplePackName}`;
}
const examplePackName = autoNameExtensionPack(
databaseItem.name,
databaseItem.language,
);
const packName = await window.showInputBox(
{
@@ -283,7 +288,7 @@ async function pickNewExtensionPack(
return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
}
const packPath = join(workspaceFolder.path, matches.groups.name);
const packPath = join(workspaceFolder.uri.fsPath, matches.groups.name);
if (await pathExists(packPath)) {
return `A pack already exists at ${packPath}`;
}
@@ -303,12 +308,145 @@ async function pickNewExtensionPack(
}
const name = matches.groups.name;
const packPath = join(workspaceFolder.path, name);
const packPath = join(workspaceFolder.uri.fsPath, name);
if (await pathExists(packPath)) {
return undefined;
}
return writeExtensionPack(packPath, packName, databaseItem.language);
}
async function autoCreateExtensionPack(
name: string,
language: string,
extensionPacksInfo: QlpacksInfo,
logger: NotificationLogger,
): Promise<ExtensionPack | undefined> {
const workspaceFolder = await autoPickWorkspaceFolder(language);
if (!workspaceFolder) {
return undefined;
}
const packName = autoNameExtensionPack(name, language);
if (!packName) {
void showAndLogErrorMessage(
logger,
`Could not automatically name extension pack for database ${name}`,
);
return undefined;
}
const existingExtensionPackPaths = extensionPacksInfo[packName];
if (existingExtensionPackPaths?.length === 1) {
let extensionPack: ExtensionPack;
try {
extensionPack = await readExtensionPack(existingExtensionPackPaths[0]);
} catch (e: unknown) {
void showAndLogErrorMessage(
logger,
`Could not read extension pack ${packName}`,
{
fullMessage: `Could not read extension pack ${packName} at ${
existingExtensionPackPaths[0]
}: ${getErrorMessage(e)}`,
},
);
return undefined;
}
return extensionPack;
} else if (existingExtensionPackPaths?.length > 1) {
void showAndLogErrorMessage(
logger,
`Extension pack ${packName} resolves to multiple paths`,
{
fullMessage: `Extension pack ${packName} resolves to multiple paths: ${existingExtensionPackPaths.join(
", ",
)}`,
},
);
return undefined;
}
const matches = packNameRegex.exec(packName);
if (!matches?.groups) {
void showAndLogErrorMessage(
logger,
`Extension pack ${packName} does not have a valid name`,
);
return undefined;
}
const unscopedName = matches.groups.name;
const packPath = join(workspaceFolder.uri.fsPath, unscopedName);
if (await pathExists(packPath)) {
void showAndLogErrorMessage(
logger,
`Directory ${packPath} already exists for extension pack ${packName}`,
);
return undefined;
}
return writeExtensionPack(packPath, packName, language);
}
async function autoPickWorkspaceFolder(
language: string,
): Promise<WorkspaceFolder | undefined> {
const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
if (workspaceFolders.length === 1) {
return workspaceFolders[0];
}
const starterWorkspaceFolderForLanguage = workspaceFolders.find(
(folder) => folder.name === `codeql-custom-queries-${language}`,
);
if (starterWorkspaceFolderForLanguage) {
return starterWorkspaceFolderForLanguage;
}
const workspaceFolderForLanguage = workspaceFolders.find((folder) =>
folder.name.endsWith(`-${language}`),
);
if (workspaceFolderForLanguage) {
return workspaceFolderForLanguage;
}
return askForWorkspaceFolder();
}
async function askForWorkspaceFolder(): Promise<WorkspaceFolder | undefined> {
const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
const workspaceFolderOptions = workspaceFolders.map((folder) => ({
label: folder.name,
detail: folder.uri.fsPath,
folder,
}));
// We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
// we only want to include on-disk workspace folders.
const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
title: "Select workspace folder to create extension pack in",
});
if (!workspaceFolder) {
return undefined;
}
return workspaceFolder.folder;
}
async function writeExtensionPack(
packPath: string,
packName: string,
language: string,
): Promise<ExtensionPack> {
const packYamlPath = join(packPath, "codeql-pack.yml");
const extensionPack: ExtensionPack = {
@@ -317,7 +455,7 @@ async function pickNewExtensionPack(
name: packName,
version: "0.0.0",
extensionTargets: {
[`codeql/${databaseItem.language}-all`]: "*",
[`codeql/${language}-all`]: "*",
},
dataExtensions: ["models/**/*.yml"],
};
@@ -420,3 +558,40 @@ async function readExtensionPack(path: string): Promise<ExtensionPack> {
dataExtensions,
};
}
function autoNameExtensionPack(
name: string,
language: string,
): string | undefined {
let packName = `${name}-${language}`;
if (!packName.includes("/")) {
packName = `pack/${packName}`;
}
const parts = packName.split("/");
const sanitizedParts = parts.map((part) => sanitizeExtensionPackName(part));
// This will ensure there's only 1 slash
packName = `${sanitizedParts[0]}/${sanitizedParts.slice(1).join("-")}`;
return packName;
}
function sanitizeExtensionPackName(name: string) {
// Lowercase everything
name = name.toLowerCase();
// Replace all spaces, dots, and underscores with hyphens
name = name.replaceAll(/[\s._]+/g, "-");
// Replace all characters which are not allowed by empty strings
name = name.replaceAll(/[^a-z0-9-]/g, "");
// Remove any leading or trailing hyphens
name = name.replaceAll(/^-|-$/g, "");
// Remove any duplicate hyphens
name = name.replaceAll(/-{2,}/g, "-");
return name;
}

229
extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts
View File

@@ -1,4 +1,11 @@
import { CancellationTokenSource, QuickPickItem, window } from "vscode";
import {
CancellationTokenSource,
QuickPickItem,
Uri,
window,
workspace,
WorkspaceFolder,
} from "vscode";
import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import { outputFile, readFile } from "fs-extra";
import { join } from "path";
@@ -8,6 +15,8 @@ import {
ResolveExtensionsResult,
} from "../../../../src/codeql-cli/cli";
import * as config from "../../../../src/config";
import { pickExtensionPackModelFile } from "../../../../src/data-extensions-editor/extension-pack-picker";
import { ExtensionPack } from "../../../../src/data-extensions-editor/shared/extension-pack";
import { createMockLogger } from "../../../__mocks__/loggerMock";
@@ -16,8 +25,10 @@ describe("pickExtensionPackModelFile", () => {
let tmpDir: string;
let extensionPackPath: string;
let anotherExtensionPackPath: string;
let autoExtensionPackPath: string;
let extensionPack: ExtensionPack;
let anotherExtensionPack: ExtensionPack;
let autoExtensionPack: ExtensionPack;
let qlPacks: QlpacksInfo;
let extensions: ResolveExtensionsResult;
@@ -32,6 +43,12 @@ describe("pickExtensionPackModelFile", () => {
const progress = jest.fn();
let showQuickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
let showInputBoxSpy: jest.SpiedFunction<typeof window.showInputBox>;
let disableAutoNameExtensionPackSpy: jest.SpiedFunction<
typeof config.disableAutoNameExtensionPack
>;
let workspaceFoldersSpy: jest.SpyInstance;
let additionalPacks: string[];
let workspaceFolder: WorkspaceFolder;
const logger = createMockLogger();
@@ -44,10 +61,12 @@ describe("pickExtensionPackModelFile", () => {
extensionPackPath = join(tmpDir, "my-extension-pack");
anotherExtensionPackPath = join(tmpDir, "another-extension-pack");
autoExtensionPackPath = join(tmpDir, "vscode-codeql-java");
qlPacks = {
"my-extension-pack": [extensionPackPath],
"another-extension-pack": [anotherExtensionPackPath],
"github/vscode-codeql-java": [autoExtensionPackPath],
};
extensions = {
models: [],
@@ -59,6 +78,13 @@ describe("pickExtensionPackModelFile", () => {
predicate: "sinkModel",
},
],
[autoExtensionPackPath]: [
{
file: join(autoExtensionPackPath, "models", "model.yml"),
index: 0,
predicate: "sinkModel",
},
],
},
};
@@ -70,6 +96,10 @@ describe("pickExtensionPackModelFile", () => {
anotherExtensionPackPath,
"another-extension-pack",
);
autoExtensionPack = await createMockExtensionPack(
autoExtensionPackPath,
"github/vscode-codeql-java",
);
showQuickPickSpy = jest
.spyOn(window, "showQuickPick")
@@ -77,6 +107,19 @@ describe("pickExtensionPackModelFile", () => {
showInputBoxSpy = jest
.spyOn(window, "showInputBox")
.mockRejectedValue(new Error("Unexpected call to showInputBox"));
disableAutoNameExtensionPackSpy = jest
.spyOn(config, "disableAutoNameExtensionPack")
.mockReturnValue(true);
workspaceFolder = {
uri: Uri.file(tmpDir),
name: "codeql-custom-queries-java",
index: 0,
};
additionalPacks = [tmpDir];
workspaceFoldersSpy = jest
.spyOn(workspace, "workspaceFolders", "get")
.mockReturnValue([workspaceFolder]);
});
it("allows choosing an existing extension pack and model file", async () => {
@@ -120,6 +163,12 @@ describe("pickExtensionPackModelFile", () => {
detail: anotherExtensionPackPath,
extensionPack: anotherExtensionPack,
},
{
label: "github/vscode-codeql-java",
description: "0.0.0",
detail: autoExtensionPackPath,
extensionPack: autoExtensionPack,
},
{
label: expect.stringMatching(/create/i),
extensionPack: null,
@@ -147,11 +196,14 @@ describe("pickExtensionPackModelFile", () => {
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith(
additionalPacks,
true,
);
expect(cliServer.resolveExtensions).toHaveBeenCalledTimes(1);
expect(cliServer.resolveExtensions).toHaveBeenCalledWith(
extensionPackPath,
[],
additionalPacks,
);
});
@@ -190,14 +242,148 @@ describe("pickExtensionPackModelFile", () => {
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith(
additionalPacks,
true,
);
expect(cliServer.resolveExtensions).toHaveBeenCalledTimes(1);
expect(cliServer.resolveExtensions).toHaveBeenCalledWith(
extensionPackPath,
[],
additionalPacks,
);
});
it("automatically selects an extension pack and allows selecting an existing model file", async () => {
disableAutoNameExtensionPackSpy.mockReturnValue(false);
const modelPath = join(autoExtensionPackPath, "models", "model.yml");
const cliServer = mockCliServer(qlPacks, extensions);
showQuickPickSpy.mockResolvedValueOnce({
label: "models/model.yml",
file: modelPath,
} as QuickPickItem);
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
logger,
progress,
token,
),
).toEqual({
filename: modelPath,
extensionPack: autoExtensionPack,
});
expect(showQuickPickSpy).toHaveBeenCalledTimes(1);
expect(showQuickPickSpy).toHaveBeenCalledWith(
[
{
label: "models/model.yml",
file: modelPath,
},
{
label: expect.stringMatching(/create/i),
file: null,
},
],
{
title: expect.any(String),
},
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith(
additionalPacks,
true,
);
expect(cliServer.resolveExtensions).toHaveBeenCalledTimes(1);
expect(cliServer.resolveExtensions).toHaveBeenCalledWith(
autoExtensionPackPath,
additionalPacks,
);
});
it("automatically creates an extension pack and allows creating a new model file", async () => {
disableAutoNameExtensionPackSpy.mockReturnValue(false);
const tmpDir = await dir({
unsafeCleanup: true,
});
workspaceFoldersSpy.mockReturnValue([
{
uri: Uri.file("/b/a/c"),
name: "my-workspace",
index: 0,
},
{
uri: Uri.file("/a/b/c"),
name: "codeql-custom-queries-csharp",
index: 1,
},
{
uri: Uri.file(tmpDir.path),
name: "codeql-custom-queries-java",
index: 2,
},
]);
const newPackDir = join(tmpDir.path, "vscode-codeql-java");
const cliServer = mockCliServer({}, { models: [], data: {} });
showInputBoxSpy.mockResolvedValue("models/my-model.yml");
expect(
await pickExtensionPackModelFile(
cliServer,
databaseItem,
logger,
progress,
token,
),
).toEqual({
filename: join(newPackDir, "models", "my-model.yml"),
extensionPack: {
path: newPackDir,
yamlPath: join(newPackDir, "codeql-pack.yml"),
name: "github/vscode-codeql-java",
version: "0.0.0",
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
},
});
expect(showQuickPickSpy).not.toHaveBeenCalled();
expect(showInputBoxSpy).toHaveBeenCalledTimes(1);
expect(showInputBoxSpy).toHaveBeenCalledWith(
{
title: expect.stringMatching(/model file/),
value: "models/github.vscode-codeql.model.yml",
validateInput: expect.any(Function),
},
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
expect(cliServer.resolveExtensions).toHaveBeenCalled();
expect(
loadYaml(await readFile(join(newPackDir, "codeql-pack.yml"), "utf8")),
).toEqual({
name: "github/vscode-codeql-java",
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
});
});
it("allows cancelling the extension pack prompt", async () => {
const cliServer = mockCliServer(qlPacks, extensions);
@@ -227,7 +413,11 @@ describe("pickExtensionPackModelFile", () => {
showQuickPickSpy.mockResolvedValueOnce({
label: "codeql-custom-queries-java",
path: tmpDir.path,
folder: {
uri: Uri.file(tmpDir.path),
name: "codeql-custom-queries-java",
index: 0,
},
} as QuickPickItem);
showInputBoxSpy.mockResolvedValueOnce("pack/new-extension-pack");
showInputBoxSpy.mockResolvedValue("models/my-model.yml");
@@ -259,7 +449,7 @@ describe("pickExtensionPackModelFile", () => {
{
title: expect.stringMatching(/extension pack/i),
prompt: expect.stringMatching(/extension pack/i),
placeHolder: expect.stringMatching(/github\/vscode-codeql-extensions/),
placeHolder: expect.stringMatching(/github\/vscode-codeql-java/),
validateInput: expect.any(Function),
},
token,
@@ -299,7 +489,11 @@ describe("pickExtensionPackModelFile", () => {
showQuickPickSpy.mockResolvedValueOnce({
label: "codeql-custom-queries-java",
path: tmpDir.path,
folder: {
uri: Uri.file(tmpDir.path),
name: "codeql-custom-queries-java",
index: 0,
},
} as QuickPickItem);
showInputBoxSpy.mockResolvedValueOnce("pack/new-extension-pack");
showInputBoxSpy.mockResolvedValue("models/my-model.yml");
@@ -334,7 +528,7 @@ describe("pickExtensionPackModelFile", () => {
{
title: expect.stringMatching(/extension pack/i),
prompt: expect.stringMatching(/extension pack/i),
placeHolder: expect.stringMatching(/github\/vscode-codeql-extensions/),
placeHolder: expect.stringMatching(/github\/vscode-codeql-csharp/),
validateInput: expect.any(Function),
},
token,
@@ -388,7 +582,11 @@ describe("pickExtensionPackModelFile", () => {
showQuickPickSpy.mockResolvedValueOnce({
label: "codeql-custom-queries-java",
path: "/a/b/c",
folder: {
uri: Uri.file("/a/b/c"),
name: "codeql-custom-queries-java",
index: 0,
},
} as QuickPickItem);
showInputBoxSpy.mockResolvedValueOnce(undefined);
@@ -777,7 +975,11 @@ describe("pickExtensionPackModelFile", () => {
showQuickPickSpy.mockResolvedValueOnce({
label: "a",
path: "/a/b/c",
folder: {
uri: Uri.file("/a/b/c"),
name: "a",
index: 0,
},
} as QuickPickItem);
showInputBoxSpy.mockResolvedValue(undefined);
@@ -1014,7 +1216,10 @@ describe("pickExtensionPackModelFile", () => {
token,
);
expect(cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
expect(cliServer.resolveQlpacks).toHaveBeenCalledWith(
additionalPacks,
true,
);
expect(cliServer.resolveExtensions).not.toHaveBeenCalled();
});
});