mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

WIP: standalone MRVA

Browse Source
This commit is contained in:
Nicolas Will
2024-07-01 18:20:23 +02:00
parent d4df484acb
commit d40cda150c
14 changed files with 163 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.vscode/settings.json vendored
View File

@@ -111,4 +111,10 @@
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true,
},
"github.copilot.advanced": {
},
"codeQL.variantAnalysis.enableGhecDr": true,
"github-enterprise.uri": "http://localhost:8080/"
}

12
extensions/ql-vscode/package.json
View File

@@ -339,13 +339,6 @@
"title": "Variant analysis",
"order": 5,
"properties": {
"codeQL.variantAnalysis.controllerRepo": {
"type": "string",
"default": "",
"pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
"patternErrorMessage": "Please enter a valid GitHub repository",
"markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
},
"codeQL.variantAnalysis.defaultResultsFilter": {
"type": "string",
"default": "all",
@@ -1931,11 +1924,6 @@
{
"view": "codeQLEvalLogViewer",
"contents": "Run the 'Show Evaluator Log (UI)' command on a CodeQL query run in the Query History view."
},
{
"view": "codeQLVariantAnalysisRepositories",
"contents": "Set up a controller repository to start using variant analysis. [Learn more](https://codeql.github.com/docs/codeql-for-visual-studio-code/running-codeql-queries-at-scale-with-mrva#controller-repository) about controller repositories. \n[Set up controller repository](command:codeQLVariantAnalysisRepositories.setupControllerRepository)",
"when": "!config.codeQL.variantAnalysis.controllerRepo"
}
]
},

1
extensions/ql-vscode/src/common/commands.ts
View File

@@ -290,7 +290,6 @@ export type DatabasePanelCommands = {
"codeQLVariantAnalysisRepositories.openConfigFile": () => Promise<void>;
"codeQLVariantAnalysisRepositories.addNewDatabase": () => Promise<void>;
"codeQLVariantAnalysisRepositories.addNewList": () => Promise<void>;
"codeQLVariantAnalysisRepositories.setupControllerRepository": () => Promise<void>;
"codeQLVariantAnalysisRepositories.setSelectedItem": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
"codeQLVariantAnalysisRepositories.setSelectedItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;

28
extensions/ql-vscode/src/config.ts
View File

@@ -112,7 +112,9 @@ export function hasEnterpriseUri(): boolean {
* Does the uri look like GHEC-DR?
*/
function isGhecDrUri(uri: Uri | undefined): boolean {
return uri !== undefined && uri.authority.toLowerCase().endsWith(".ghe.com");
return (
uri !== undefined && !uri.authority.toLowerCase().endsWith("github.com")
);
}
/**
@@ -591,27 +593,7 @@ export const NO_CACHE_CONTEXTUAL_QUERIES = new Setting(
// Settings for variant analysis
const VARIANT_ANALYSIS_SETTING = new Setting("variantAnalysis", ROOT_SETTING);
/**
* The name of the "controller" repository that you want to use with the "Run Variant Analysis" command.
* Note: This command is only available for internal users.
*
* This setting should be a GitHub repository of the form `<owner>/<repo>`.
*/
const REMOTE_CONTROLLER_REPO = new Setting(
"controllerRepo",
VARIANT_ANALYSIS_SETTING,
);
export function getRemoteControllerRepo(): string | undefined {
return REMOTE_CONTROLLER_REPO.getValue<string>() || undefined;
}
export async function setRemoteControllerRepo(repo: string | undefined) {
await REMOTE_CONTROLLER_REPO.updateValue(repo, ConfigurationTarget.Global);
}
export interface VariantAnalysisConfig {
controllerRepo: string | undefined;
showSystemDefinedRepositoryLists: boolean;
/**
* This uses a URL instead of a URI because the URL class is available in
@@ -632,10 +614,6 @@ export class VariantAnalysisConfigListener
);
}
public get controllerRepo(): string | undefined {
return getRemoteControllerRepo();
}
public get showSystemDefinedRepositoryLists(): boolean {
return !hasEnterpriseUri();
}

23
extensions/ql-vscode/src/databases/ui/db-panel.ts
View File

@@ -18,8 +18,6 @@ import type { DbManager } from "../db-manager";
import { DbTreeDataProvider } from "./db-tree-data-provider";
import type { DbTreeViewItem } from "./db-tree-view-item";
import { getGitHubUrl } from "./db-tree-view-item-action";
import { getControllerRepo } from "../../variant-analysis/run-remote-query";
import { getErrorMessage } from "../../common/helpers-pure";
import type { DatabasePanelCommands } from "../../common/commands";
import type { App } from "../../common/app";
import { QueryLanguage } from "../../common/query-language";
@@ -74,9 +72,6 @@ export class DbPanel extends DisposableObject {
this.addNewRemoteDatabase.bind(this),
"codeQLVariantAnalysisRepositories.addNewList":
this.addNewList.bind(this),
"codeQLVariantAnalysisRepositories.setupControllerRepository":
this.setupControllerRepository.bind(this),
"codeQLVariantAnalysisRepositories.setSelectedItem":
this.setSelectedItem.bind(this),
"codeQLVariantAnalysisRepositories.setSelectedItemContextMenu":
@@ -427,22 +422,4 @@ export class DbPanel extends DisposableObject {
await this.app.commands.execute("vscode.open", Uri.parse(githubUrl));
}
private async setupControllerRepository(): Promise<void> {
try {
// This will also validate that the controller repository is valid
await getControllerRepo(this.app.credentials);
} catch (e: unknown) {
if (e instanceof UserCancellationException) {
return;
}
void showAndLogErrorMessage(
this.app.logger,
`An error occurred while setting up the controller repository: ${getErrorMessage(
e,
)}`,
);
}
}
}

5
extensions/ql-vscode/src/databases/ui/db-tree-data-provider.ts
View File

@@ -83,11 +83,6 @@ export class DbTreeDataProvider
}
private createTree(): DbTreeViewItem[] {
// Returning an empty tree here will show the welcome view
if (!this.variantAnalysisConfig.controllerRepo) {
return [];
}
const dbItemsResult = this.dbManager.getDbItems();
if (dbItemsResult.isFailure) {

5
extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-domain-mapper.ts
View File

@@ -48,11 +48,6 @@ function mapVariantAnalysisDtoToDto(
): VariantAnalysisDto {
return {
id: variantAnalysis.id,
controllerRepo: {
id: variantAnalysis.controllerRepo.id,
fullName: variantAnalysis.controllerRepo.fullName,
private: variantAnalysis.controllerRepo.private,
},
query: {
name: variantAnalysis.query.name,
filePath: variantAnalysis.query.filePath,

10
extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto-mapper.ts
View File

@@ -48,12 +48,12 @@ function mapVariantAnalysisToDomainModel(
): VariantAnalysis {
return {
id: variantAnalysis.id,
controllerRepo: {
id: variantAnalysis.controllerRepo.id,
fullName: variantAnalysis.controllerRepo.fullName,
private: variantAnalysis.controllerRepo.private,
},
language: mapQueryLanguageToDomainModel(variantAnalysis.query.language),
controllerRepo: {
id: 0,
fullName: "",
private: false,
},
query: {
name: variantAnalysis.query.name,
filePath: variantAnalysis.query.filePath,

5
extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-dto.ts
View File

@@ -15,11 +15,6 @@ export interface QueryHistoryVariantAnalysisDto {
export interface VariantAnalysisDto {
id: number;
controllerRepo: {
id: number;
fullName: string;
private: boolean;
};
query: {
name: string;
filePath: string;

8
extensions/ql-vscode/src/variant-analysis/export-results.ts
View File

@@ -160,7 +160,7 @@ async function exportVariantAnalysisAnalysisResults(
expectedAnalysesResultsCount: number,
exportFormat: "gist" | "local",
commandManager: AppCommandManager,
credentials: Credentials,
_credentials: Credentials,
progress: ProgressCallback,
token: CancellationToken,
) {
@@ -191,7 +191,6 @@ async function exportVariantAnalysisAnalysisResults(
markdownFiles,
exportFormat,
commandManager,
credentials,
progress,
token,
);
@@ -236,7 +235,6 @@ async function exportResults(
markdownFiles: MarkdownFile[],
exportFormat: "gist" | "local",
commandManager: AppCommandManager,
credentials: Credentials,
progress?: ProgressCallback,
token?: CancellationToken,
) {
@@ -249,7 +247,6 @@ async function exportResults(
description,
markdownFiles,
commandManager,
credentials,
progress,
token,
);
@@ -268,7 +265,6 @@ async function exportToGist(
description: string,
markdownFiles: MarkdownFile[],
commandManager: AppCommandManager,
credentials: Credentials,
progress?: ProgressCallback,
token?: CancellationToken,
) {
@@ -291,7 +287,7 @@ async function exportToGist(
{} as { [key: string]: { content: string } },
);
const gistUrl = await createGist(credentials, description, gistFiles);
const gistUrl = await createGist(description, gistFiles);
if (gistUrl) {
// This needs to use .then to ensure we aren't keeping the progress notification open. We shouldn't await the
// "Open gist" button click.

140
extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
View File

@@ -1,5 +1,4 @@
import type { OctokitResponse } from "@octokit/types/dist-types";
import type { Credentials } from "../../common/authentication";
import { getGitHubInstanceUrl } from "../../config";
import type { VariantAnalysisSubmission } from "../shared/variant-analysis";
import type {
VariantAnalysis,
@@ -7,12 +6,26 @@ import type {
VariantAnalysisSubmissionRequest,
} from "./variant-analysis";
import type { Repository } from "./repository";
import { extLogger } from "../../common/logging/vscode";
function getOctokitBaseUrl(): string {
let apiUrl = getGitHubInstanceUrl().toString();
if (apiUrl.endsWith("/")) {
apiUrl = apiUrl.slice(0, -1);
}
if (apiUrl.startsWith("https://")) {
apiUrl = apiUrl.replace("https://", "http://");
}
return apiUrl;
}
export async function submitVariantAnalysis(
credentials: Credentials,
submissionDetails: VariantAnalysisSubmission,
): Promise<VariantAnalysis> {
const octokit = await credentials.getOctokit();
try {
console.log("Getting base URL...");
const baseUrl = getOctokitBaseUrl();
void extLogger.log(`Base URL: ${baseUrl}`);
const { actionRepoRef, language, pack, databases, controllerRepoId } =
submissionDetails;
@@ -26,65 +39,109 @@ export async function submitVariantAnalysis(
repository_owners: databases.repositoryOwners,
};
const response: OctokitResponse<VariantAnalysis> = await octokit.request(
"POST /repositories/:controllerRepoId/code-scanning/codeql/variant-analyses",
void extLogger.log(
`Sending fetch request with data: ${JSON.stringify(data)}`,
);
void extLogger.log(
`Fetch request URL: ${baseUrl}/repositories/${controllerRepoId}/code-scanning/codeql/variant-analyses`,
);
const response = await fetch(
`${baseUrl}/repositories/${controllerRepoId}/code-scanning/codeql/variant-analyses`,
{
controllerRepoId,
data,
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(data),
},
);
return response.data;
}
void extLogger.log(`Response status: ${response.status}`);
if (!response.ok) {
throw new Error(
`Error submitting variant analysis: ${response.statusText}`,
);
}
const responseData = await response.json();
void extLogger.log(`Response data: ${responseData}`);
return responseData;
} catch (error) {
void extLogger.log(`Error: ${error}`);
throw error;
}
}
export async function getVariantAnalysis(
credentials: Credentials,
controllerRepoId: number,
variantAnalysisId: number,
): Promise<VariantAnalysis> {
const octokit = await credentials.getOctokit();
const baseUrl = getOctokitBaseUrl();
const response: OctokitResponse<VariantAnalysis> = await octokit.request(
"GET /repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId",
const response = await fetch(
`${baseUrl}/repositories/${controllerRepoId}/code-scanning/codeql/variant-analyses/${variantAnalysisId}`,
{
controllerRepoId,
variantAnalysisId,
method: "GET",
headers: {
"Content-Type": "application/json",
},
},
);
return response.data;
if (!response.ok) {
throw new Error(`Error getting variant analysis: ${response.statusText}`);
}
return response.json();
}
export async function getVariantAnalysisRepo(
credentials: Credentials,
controllerRepoId: number,
variantAnalysisId: number,
repoId: number,
): Promise<VariantAnalysisRepoTask> {
const octokit = await credentials.getOctokit();
const baseUrl = getOctokitBaseUrl();
const response: OctokitResponse<VariantAnalysisRepoTask> =
await octokit.request(
"GET /repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId/repositories/:repoId",
const response = await fetch(
`${baseUrl}/repositories/${controllerRepoId}/code-scanning/codeql/variant-analyses/${variantAnalysisId}/repositories/${repoId}`,
{
controllerRepoId,
variantAnalysisId,
repoId,
method: "GET",
headers: {
"Content-Type": "application/json",
},
},
);
return response.data;
if (!response.ok) {
throw new Error(
`Error getting variant analysis repo: ${response.statusText}`,
);
}
return response.json();
}
export async function getRepositoryFromNwo(
credentials: Credentials,
owner: string,
repo: string,
): Promise<Repository> {
const octokit = await credentials.getOctokit();
const baseUrl = getOctokitBaseUrl();
const response = await octokit.rest.repos.get({ owner, repo });
return response.data as Repository;
const response = await fetch(`${baseUrl}/repos/${owner}/${repo}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
},
});
if (!response.ok) {
throw new Error(`Error getting repository: ${response.statusText}`);
}
return response.json();
}
/**
@@ -92,22 +149,29 @@ export async function getRepositoryFromNwo(
* Returns the URL of the created gist.
*/
export async function createGist(
credentials: Credentials,
description: string,
files: { [key: string]: { content: string } },
): Promise<string | undefined> {
const octokit = await credentials.getOctokit();
const response = await octokit.request("POST /gists", {
const baseUrl = getOctokitBaseUrl();
const response = await fetch(`${baseUrl}/gists`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
description,
files,
public: false,
}),
});
if (response.status >= 300) {
if (!response.ok) {
throw new Error(
`Error exporting variant analysis results: ${response.status} ${
response?.data || ""
}`,
`Error creating gist: ${response.status} ${response.statusText}`,
);
}
return response.data.html_url;
const data = await response.json();
return data.html_url;
}

96
extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
View File

@@ -1,5 +1,5 @@
import type { CancellationToken } from "vscode";
import { Uri, window } from "vscode";
import { Uri } from "vscode";
import { join, sep, basename, relative } from "path";
import { dump, load } from "js-yaml";
import { copy, writeFile, readFile, mkdirp } from "fs-extra";
@@ -7,26 +7,17 @@ import type { DirectoryResult } from "tmp-promise";
import { dir, tmpName } from "tmp-promise";
import { tmpDir } from "../tmp-dir";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import type { Credentials } from "../common/authentication";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import { extLogger } from "../common/logging/vscode";
import {
getActionBranch,
getRemoteControllerRepo,
setRemoteControllerRepo,
} from "../config";
import { getActionBranch } from "../config";
import type { ProgressCallback } from "../common/vscode/progress";
import { UserCancellationException } from "../common/vscode/progress";
import type { RequestError } from "@octokit/types/dist-types";
import type { QueryMetadata } from "../common/interface-types";
import { getErrorMessage, REPO_REGEX } from "../common/helpers-pure";
import { getRepositoryFromNwo } from "./gh-api/gh-api-client";
import type { RepositorySelection } from "./repository-selection";
import {
getRepositorySelection,
isValidSelection,
} from "./repository-selection";
import type { Repository } from "./shared/repository";
import type { DbManager } from "../databases/db-manager";
import {
getQlPackFilePath,
@@ -285,13 +276,11 @@ interface PreparedRemoteQuery {
base64Pack: string;
modelPacks: ModelPackDetails[];
repoSelection: RepositorySelection;
controllerRepo: Repository;
queryStartTime: number;
}
export async function prepareRemoteQueryRun(
cliServer: CodeQLCliServer,
credentials: Credentials,
qlPackDetails: QlPackDetails,
progress: ProgressCallback,
token: CancellationToken,
@@ -322,8 +311,6 @@ export async function prepareRemoteQueryRun(
message: "Determining controller repo",
});
const controllerRepo = await getControllerRepo(credentials);
progress({
maxStep: 4,
step: 3,
@@ -367,7 +354,6 @@ export async function prepareRemoteQueryRun(
base64Pack: generatedPack.base64Pack,
modelPacks: generatedPack.modelPacks,
repoSelection,
controllerRepo,
queryStartTime,
};
}
@@ -494,84 +480,6 @@ export function getQueryName(
return queryMetadata?.name ?? basename(queryFilePath);
}
export async function getControllerRepo(
credentials: Credentials,
): Promise<Repository> {
// Get the controller repo from the config, if it exists.
// If it doesn't exist, prompt the user to enter it, check
// whether the repo exists, and save the nwo to the config.
let shouldSetControllerRepo = false;
let controllerRepoNwo: string | undefined;
controllerRepoNwo = getRemoteControllerRepo();
if (!controllerRepoNwo || !REPO_REGEX.test(controllerRepoNwo)) {
void extLogger.log(
controllerRepoNwo
? "Invalid controller repository name."
: "No controller repository defined.",
);
controllerRepoNwo = await window.showInputBox({
title:
"Controller repository in which to run GitHub Actions workflows for variant analyses",
placeHolder: "<owner>/<repo>",
prompt:
"Enter the name of a GitHub repository in the format <owner>/<repo>. You can change this in the extension settings.",
ignoreFocusOut: true,
});
if (!controllerRepoNwo) {
throw new UserCancellationException("No controller repository entered.");
} else if (!REPO_REGEX.test(controllerRepoNwo)) {
// Check if user entered invalid input
throw new UserCancellationException(
"Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.",
);
}
shouldSetControllerRepo = true;
}
void extLogger.log(`Using controller repository: ${controllerRepoNwo}`);
const controllerRepo = await getControllerRepoFromApi(
credentials,
controllerRepoNwo,
);
if (shouldSetControllerRepo) {
void extLogger.log(
`Setting the controller repository as: ${controllerRepoNwo}`,
);
await setRemoteControllerRepo(controllerRepoNwo);
}
return controllerRepo;
}
async function getControllerRepoFromApi(
credentials: Credentials,
nwo: string,
): Promise<Repository> {
const [owner, repo] = nwo.split("/");
try {
const controllerRepo = await getRepositoryFromNwo(credentials, owner, repo);
void extLogger.log(`Controller repository ID: ${controllerRepo.id}`);
return {
id: controllerRepo.id,
fullName: controllerRepo.full_name,
private: controllerRepo.private,
};
} catch (e) {
if ((e as RequestError).status === 404) {
throw new Error(`Controller repository "${owner}/${repo}" not found`);
} else {
throw new Error(
`Error getting controller repository "${owner}/${repo}": ${getErrorMessage(
e,
)}`,
);
}
}
}
function removeWorkspaceRefs(qlpack: QlPackFile) {
if (!qlpack.dependencies) {
return;

24
extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
View File

@@ -373,16 +373,19 @@ export class VariantAnalysisManager
);
}
// log to extLogger
void this.app.logger.log(
`Running variant analysis with query: ${queryName}, language: ${variantAnalysisLanguage}`,
);
const {
actionBranch,
base64Pack,
modelPacks,
repoSelection,
controllerRepo,
queryStartTime,
} = await prepareRemoteQueryRun(
this.cliServer,
this.app.credentials,
qlPackDetails,
progress,
token,
@@ -399,12 +402,15 @@ export class VariantAnalysisManager
count: qlPackDetails.queryFiles.length,
};
// log that submitting
void this.app.logger.log("Submitting variant analysis");
const variantAnalysisSubmission: VariantAnalysisSubmission = {
startTime: queryStartTime,
actionRepoRef: actionBranch,
controllerRepoId: controllerRepo.id,
language: variantAnalysisLanguage,
pack: base64Pack,
controllerRepoId: 0,
query: {
name: queryName,
filePath: firstQueryFile,
@@ -422,7 +428,6 @@ export class VariantAnalysisManager
let variantAnalysisResponse: ApiVariantAnalysis;
try {
variantAnalysisResponse = await submitVariantAnalysis(
this.app.credentials,
variantAnalysisSubmission,
);
} catch (e: unknown) {
@@ -431,9 +436,17 @@ export class VariantAnalysisManager
e instanceof RequestError &&
handleRequestError(e, this.config.githubUrl, this.app.logger)
) {
// log
void this.app.logger.log(
`Error submitting variant analysis: ${getErrorMessage(e)}`,
);
return undefined;
}
// throwing
void this.app.logger.log(
`Error submitting variant analysis: ${getErrorMessage(e)}`,
);
throw e;
}
@@ -806,8 +819,7 @@ export class VariantAnalysisManager
let repoTask: VariantAnalysisRepositoryTask;
try {
const repoTaskResponse = await getVariantAnalysisRepo(
this.app.credentials,
variantAnalysis.controllerRepo.id,
0,
variantAnalysis.id,
scannedRepo.repository.id,
);

8
extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
View File

@@ -62,7 +62,6 @@ export class VariantAnalysisMonitor extends DisposableObject {
try {
await this._monitorVariantAnalysis(
variantAnalysis.id,
variantAnalysis.controllerRepo.id,
variantAnalysis.executionStartTime,
variantAnalysis.query.name,
variantAnalysis.language,
@@ -74,7 +73,6 @@ export class VariantAnalysisMonitor extends DisposableObject {
private async _monitorVariantAnalysis(
variantAnalysisId: number,
controllerRepoId: number,
executionStartTime: number,
queryName: string,
language: QueryLanguage,
@@ -97,11 +95,7 @@ export class VariantAnalysisMonitor extends DisposableObject {
let variantAnalysisSummary: ApiVariantAnalysis;
try {
variantAnalysisSummary = await getVariantAnalysis(
this.app.credentials,
controllerRepoId,
variantAnalysisId,
);
variantAnalysisSummary = await getVariantAnalysis(0, variantAnalysisId);
} catch (e) {
const errorMessage = getErrorMessage(e);