mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2439 from github/nora/search-prompt

Browse Source 
Feed code search results into variant analysis repo lists
This commit is contained in:
Nora
2023-05-31 17:08:44 +02:00
committed by GitHub
parent 0b6bcfdfd6 0739c46fed
commit 157210ffc5
11 changed files with 402 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
extensions/ql-vscode/package.json
View File

@@ -516,6 +516,10 @@
"title": "Add new list",
"icon": "$(new-folder)"
},
{
"command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
"title": "Add repositories with GitHub Code Search"
},
{
"command": "codeQLVariantAnalysisRepositories.setSelectedItem",
"title": "Select"
@@ -961,6 +965,11 @@
"when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
"group": "2_qlContextMenu@1"
},
{
"command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
"when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
"group": "2_qlContextMenu@1"
},
{
"command": "codeQLDatabases.setCurrentDatabase",
"group": "inline",
@@ -1297,6 +1306,10 @@
"command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
"when": "false"
},
{
"command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
"when": "false"
},
{
"command": "codeQLDatabases.setCurrentDatabase",
"when": "false"

1
extensions/ql-vscode/src/common/commands.ts
View File

@@ -275,6 +275,7 @@ export type DatabasePanelCommands = {
"codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
};
export type AstCfgCommands = {

44
extensions/ql-vscode/src/databases/config/db-config-store.ts
View File

@@ -147,10 +147,46 @@ export class DbConfigStore extends DisposableObject {
await this.writeConfig(config);
}
/**
* Adds a list of remote repositories to an existing repository list and removes duplicates.
* @returns a list of repositories that were not added because the list reached 1000 entries.
*/
public async addRemoteReposToList(
repoNwoList: string[],
parentList: string,
): Promise<string[]> {
if (!this.config) {
throw Error("Cannot add variant analysis repos if config is not loaded");
}
const config = cloneDbConfig(this.config);
const parent = config.databases.variantAnalysis.repositoryLists.find(
(list) => list.name === parentList,
);
if (!parent) {
throw Error(`Cannot find parent list '${parentList}'`);
}
// Remove duplicates from the list of repositories.
const newRepositoriesList = [
...new Set([...parent.repositories, ...repoNwoList]),
];
parent.repositories = newRepositoriesList.slice(0, 1000);
const truncatedRepositories = newRepositoriesList.slice(1000);
await this.writeConfig(config);
return truncatedRepositories;
}
/**
* Adds one remote repository
* @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added.
*/
public async addRemoteRepo(
repoNwo: string,
parentList?: string,
): Promise<void> {
): Promise<string[]> {
if (!this.config) {
throw Error("Cannot add variant analysis repo if config is not loaded");
}
@@ -165,6 +201,7 @@ export class DbConfigStore extends DisposableObject {
);
}
const truncatedRepositories = [];
const config = cloneDbConfig(this.config);
if (parentList) {
const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -173,12 +210,15 @@ export class DbConfigStore extends DisposableObject {
if (!parent) {
throw Error(`Cannot find parent list '${parentList}'`);
} else {
parent.repositories.push(repoNwo);
const newRepositories = [...parent.repositories, repoNwo];
parent.repositories = newRepositories.slice(0, 1000);
truncatedRepositories.push(...newRepositories.slice(1000));
}
} else {
config.databases.variantAnalysis.repositories.push(repoNwo);
}
await this.writeConfig(config);
return truncatedRepositories;
}
public async addRemoteOwner(owner: string): Promise<void> {

11
extensions/ql-vscode/src/databases/db-manager.ts
View File

@@ -101,8 +101,15 @@ export class DbManager extends DisposableObject {
public async addNewRemoteRepo(
nwo: string,
parentList?: string,
): Promise<void> {
await this.dbConfigStore.addRemoteRepo(nwo, parentList);
): Promise<string[]> {
return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
}
public async addNewRemoteReposToList(
nwoList: string[],
parentList: string,
): Promise<string[]> {
return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
}
public async addNewRemoteOwner(owner: string): Promise<void> {

106
extensions/ql-vscode/src/databases/ui/db-panel.ts
View File

@@ -1,4 +1,5 @@
import {
ProgressLocation,
QuickPickItem,
TreeView,
TreeViewExpansionEvent,
@@ -13,7 +14,10 @@ import {
getOwnerFromGitHubUrl,
isValidGitHubOwner,
} from "../../common/github-url-identifier-helper";
import { showAndLogErrorMessage } from "../../helpers";
import {
showAndLogErrorMessage,
showAndLogInformationMessage,
} from "../../helpers";
import { DisposableObject } from "../../pure/disposable-object";
import {
DbItem,
@@ -32,6 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query";
import { getErrorMessage } from "../../pure/helpers-pure";
import { DatabasePanelCommands } from "../../common/commands";
import { App } from "../../common/app";
import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client";
import { QueryLanguage } from "../../common/query-language";
export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
remoteDatabaseKind: string;
@@ -41,6 +47,10 @@ export interface AddListQuickPickItem extends QuickPickItem {
databaseKind: DbListKind;
}
export interface CodeSearchQuickPickItem extends QuickPickItem {
language: string;
}
export class DbPanel extends DisposableObject {
private readonly dataProvider: DbTreeDataProvider;
private readonly treeView: TreeView<DbTreeViewItem>;
@@ -93,6 +103,8 @@ export class DbPanel extends DisposableObject {
this.renameItem.bind(this),
"codeQLVariantAnalysisRepositories.removeItemContextMenu":
this.removeItem.bind(this),
"codeQLVariantAnalysisRepositories.importFromCodeSearch":
this.importFromCodeSearch.bind(this),
};
}
@@ -171,7 +183,14 @@ export class DbPanel extends DisposableObject {
return;
}
await this.dbManager.addNewRemoteRepo(nwo, parentList);
const truncatedRepositories = await this.dbManager.addNewRemoteRepo(
nwo,
parentList,
);
if (parentList) {
this.reportAnyTruncatedRepos(truncatedRepositories, parentList);
}
}
private async addNewRemoteOwner(): Promise<void> {
@@ -323,6 +342,89 @@ export class DbPanel extends DisposableObject {
await this.dbManager.removeDbItem(treeViewItem.dbItem);
}
private async importFromCodeSearch(
treeViewItem: DbTreeViewItem,
): Promise<void> {
if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) {
throw new Error("Please select a valid list to add code search results.");
}
const listName = treeViewItem.dbItem.listName;
const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values(
QueryLanguage,
).map((language) => ({
label: language.toString(),
alwaysShow: true,
language: language.toString(),
}));
const codeSearchLanguage =
await window.showQuickPick<CodeSearchQuickPickItem>(
languageQuickPickItems,
{
title: "Select a language for your search",
placeHolder: "Select an option",
ignoreFocusOut: true,
},
);
if (!codeSearchLanguage) {
return;
}
const codeSearchQuery = await window.showInputBox({
title: "GitHub Code Search",
prompt:
"Use [GitHub's Code Search syntax](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax), including code qualifiers, regular expressions, and boolean operations, to search for repositories.",
placeHolder: "org:github",
});
if (codeSearchQuery === undefined || codeSearchQuery === "") {
return;
}
void window.withProgress(
{
location: ProgressLocation.Notification,
title: "Searching for repositories... This might take a while",
cancellable: true,
},
async (progress, token) => {
progress.report({ increment: 10 });
const repositories = await getCodeSearchRepositories(
this.app.credentials,
`${codeSearchQuery} language:${codeSearchLanguage.language}`,
progress,
token,
);
token.onCancellationRequested(() => {
void showAndLogInformationMessage("Code search cancelled");
return;
});
progress.report({ increment: 10, message: "Processing results..." });
const truncatedRepositories =
await this.dbManager.addNewRemoteReposToList(repositories, listName);
this.reportAnyTruncatedRepos(truncatedRepositories, listName);
},
);
}
private reportAnyTruncatedRepos(
truncatedRepositories: string[],
listName: string,
) {
if (truncatedRepositories.length > 0) {
void showAndLogErrorMessage(
`Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join(
", ",
)}`,
);
}
}
private async onDidCollapseElement(
event: TreeViewExpansionEvent<DbTreeViewItem>,
): Promise<void> {

11
extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts
View File

@@ -4,7 +4,8 @@ export type DbTreeViewItemAction =
| "canBeSelected"
| "canBeRemoved"
| "canBeRenamed"
| "canBeOpenedOnGitHub";
| "canBeOpenedOnGitHub"
| "canImportCodeSearch";
export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
const actions: DbTreeViewItemAction[] = [];
@@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
if (canBeOpenedOnGitHub(dbItem)) {
actions.push("canBeOpenedOnGitHub");
}
if (canImportCodeSearch(dbItem)) {
actions.push("canImportCodeSearch");
}
return actions;
}
@@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean {
return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind);
}
function canImportCodeSearch(dbItem: DbItem): boolean {
return DbItemKind.RemoteUserDefinedList === dbItem.kind;
}
export function getGitHubUrl(dbItem: DbItem): string | undefined {
switch (dbItem.kind) {
case DbItemKind.RemoteOwner:

37
extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
View File

@@ -7,6 +7,43 @@ import {
VariantAnalysisSubmissionRequest,
} from "./variant-analysis";
import { Repository } from "./repository";
import { Progress } from "vscode";
import { CancellationToken } from "vscode-jsonrpc";
export async function getCodeSearchRepositories(
credentials: Credentials,
query: string,
progress: Progress<{
message?: string | undefined;
increment?: number | undefined;
}>,
token: CancellationToken,
): Promise<string[]> {
let nwos: string[] = [];
const octokit = await credentials.getOctokit();
for await (const response of octokit.paginate.iterator(
octokit.rest.search.repos,
{
q: query,
per_page: 100,
},
)) {
nwos.push(...response.data.map((item) => item.full_name));
// calculate progress bar: 80% of the progress bar is used for the code search
const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
// Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10
const increment =
totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
progress.report({ increment });
if (token.isCancellationRequested) {
nwos = [];
break;
}
}
return [...new Set(nwos)];
}
export async function submitVariantAnalysis(
credentials: Credentials,

107
extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
View File

@@ -241,6 +241,113 @@ describe("db config store", () => {
configStore.dispose();
});
it("should add unique remote repositories to the correct list", async () => {
// Initial set up
const dbConfig = createDbConfig({
remoteLists: [
{
name: "list1",
repositories: ["owner/repo1"],
},
],
});
const configStore = await initializeConfig(dbConfig, configPath, app);
expect(
configStore.getConfig().value.databases.variantAnalysis
.repositoryLists[0],
).toEqual({
name: "list1",
repositories: ["owner/repo1"],
});
// Add
const response = await configStore.addRemoteReposToList(
["owner/repo1", "owner/repo2"],
"list1",
);
// Read the config file
const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
// Check that the config file has been updated
const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
expect(updatedRemoteDbs.repositories).toHaveLength(0);
expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
expect(updatedRemoteDbs.repositoryLists[0]).toEqual({
name: "list1",
repositories: ["owner/repo1", "owner/repo2"],
});
expect(response).toEqual([]);
configStore.dispose();
});
it("should add no more than 1000 repositories to a remote list when adding multiple repos", async () => {
// Initial set up
const dbConfig = createDbConfig({
remoteLists: [
{
name: "list1",
repositories: [],
},
],
});
const configStore = await initializeConfig(dbConfig, configPath, app);
// Add
const response = await configStore.addRemoteReposToList(
[...Array(1001).keys()].map((i) => `owner/db${i}`),
"list1",
);
// Read the config file
const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
// Check that the config file has been updated
const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
expect(updatedRemoteDbs.repositories).toHaveLength(0);
expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
1000,
);
expect(response).toEqual(["owner/db1000"]);
configStore.dispose();
});
it("should add no more than 1000 repositories to a remote list when adding one repo", async () => {
// Initial set up
const dbConfig = createDbConfig({
remoteLists: [
{
name: "list1",
repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
},
],
});
const configStore = await initializeConfig(dbConfig, configPath, app);
// Add
const reponse = await configStore.addRemoteRepo("owner/db1000", "list1");
// Read the config file
const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
// Check that the config file has been updated
const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
expect(updatedRemoteDbs.repositories).toHaveLength(0);
expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
1000,
);
expect(reponse).toEqual(["owner/db1000"]);
configStore.dispose();
});
it("should add a remote owner", async () => {
// Initial set up
const dbConfig = createDbConfig();

67
extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
View File

@@ -88,6 +88,73 @@ describe("db manager", () => {
).toEqual("owner2/repo2");
});
it("should add new remote repos to a user defined list", async () => {
const dbConfig: DbConfig = createDbConfig({
remoteLists: [
{
name: "my-list-1",
repositories: ["owner1/repo1"],
},
],
});
await saveDbConfig(dbConfig);
await dbManager.addNewRemoteReposToList(["owner2/repo2"], "my-list-1");
const dbConfigFileContents = await readDbConfigDirectly();
expect(
dbConfigFileContents.databases.variantAnalysis.repositoryLists.length,
).toBe(1);
expect(
dbConfigFileContents.databases.variantAnalysis.repositoryLists[0],
).toEqual({
name: "my-list-1",
repositories: ["owner1/repo1", "owner2/repo2"],
});
});
it("should return truncated repos when adding multiple repos to a user defined list", async () => {
const dbConfig: DbConfig = createDbConfig({
remoteLists: [
{
name: "my-list-1",
repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
},
],
});
await saveDbConfig(dbConfig);
const response = await dbManager.addNewRemoteReposToList(
["owner2/repo2"],
"my-list-1",
);
expect(response).toEqual(["owner2/repo2"]);
});
it("should return truncated repos when adding one repo to a user defined list", async () => {
const dbConfig: DbConfig = createDbConfig({
remoteLists: [
{
name: "my-list-1",
repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
},
],
});
await saveDbConfig(dbConfig);
const response = await dbManager.addNewRemoteRepo(
"owner2/repo2",
"my-list-1",
);
expect(response).toEqual(["owner2/repo2"]);
});
it("should add a new remote repo to a user defined list", async () => {
const dbConfig: DbConfig = createDbConfig({
remoteLists: [

9
extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts
View File

@@ -62,12 +62,17 @@ describe("getDbItemActions", () => {
expect(actions.length).toEqual(0);
});
it("should set canBeSelected, canBeRemoved and canBeRenamed for remote user defined db list", () => {
it("should set canBeSelected, canBeRemoved, canBeRenamed and canImportCodeSearch for remote user defined db list", () => {
const dbItem = createRemoteUserDefinedListDbItem();
const actions = getDbItemActions(dbItem);
expect(actions).toEqual(["canBeSelected", "canBeRemoved", "canBeRenamed"]);
expect(actions).toEqual([
"canBeSelected",
"canBeRemoved",
"canBeRenamed",
"canImportCodeSearch",
]);
});
it("should not set canBeSelected for remote user defined db list that is already selected", () => {

7
extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts
View File

@@ -349,7 +349,12 @@ describe("db panel rendering nodes", () => {
expect(item.tooltip).toBeUndefined();
expect(item.iconPath).toBeUndefined();
expect(item.collapsibleState).toBe(TreeItemCollapsibleState.Collapsed);
checkDbItemActions(item, ["canBeSelected", "canBeRenamed", "canBeRemoved"]);
checkDbItemActions(item, [
"canBeSelected",
"canBeRenamed",
"canBeRemoved",
"canImportCodeSearch",
]);
expect(item.children).toBeTruthy();
expect(item.children.length).toBe(repos.length);