mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into robertbrignull/fix_alert_547

Browse Source
This commit is contained in:
Robert
2023-07-26 10:26:24 +01:00
committed by GitHub
parent 07b8732a31 e877695a14
commit c5a4c53a1a
13 changed files with 690 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
extensions/ql-vscode/src/common/interface-types.ts
View File

@@ -500,13 +500,6 @@ interface SetExternalApiUsagesMessage {
externalApiUsages: ExternalApiUsage[];
}
export interface ShowProgressMessage {
t: "showProgress";
step: number;
maxStep: number;
message: string;
}
interface LoadModeledMethodsMessage {
t: "loadModeledMethods";
modeledMethods: Record<string, ModeledMethod>;
@@ -558,7 +551,6 @@ interface ModelDependencyMessage {
export type ToDataExtensionsEditorMessage =
| SetExtensionPackStateMessage
| SetExternalApiUsagesMessage
| ShowProgressMessage
| LoadModeledMethodsMessage
| AddModeledMethodsMessage;

386
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
View File

@@ -14,11 +14,7 @@ import {
FromDataExtensionsEditorMessage,
ToDataExtensionsEditorMessage,
} from "../common/interface-types";
import {
ProgressCallback,
ProgressUpdate,
withProgress,
} from "../common/vscode/progress";
import { ProgressCallback, withProgress } from "../common/vscode/progress";
import { QueryRunner } from "../query-server";
import {
showAndLogExceptionWithTelemetry,
@@ -225,203 +221,205 @@ export class DataExtensionsEditorView extends AbstractWebview<
}
protected async loadExternalApiUsages(): Promise<void> {
const cancellationTokenSource = new CancellationTokenSource();
await withProgress(
async (progress) => {
try {
const cancellationTokenSource = new CancellationTokenSource();
const queryResult = await runQuery(
this.mode === Mode.Framework
? "frameworkModeQuery"
: "applicationModeQuery",
{
cliServer: this.cliServer,
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
queryStorageDir: this.queryStorageDir,
progress: (update) => progress({ ...update, maxStep: 1500 }),
token: cancellationTokenSource.token,
},
);
if (!queryResult) {
return;
}
try {
const queryResult = await runQuery(
this.mode === Mode.Framework
? "frameworkModeQuery"
: "applicationModeQuery",
{
cliServer: this.cliServer,
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
queryStorageDir: this.queryStorageDir,
progress: (progressUpdate: ProgressUpdate) => {
void this.showProgress(progressUpdate, 1500);
},
token: cancellationTokenSource.token,
},
);
if (!queryResult) {
await this.clearProgress();
return;
}
progress({
message: "Decoding results",
step: 1100,
maxStep: 1500,
});
await this.showProgress({
message: "Decoding results",
step: 1100,
maxStep: 1500,
});
const bqrsChunk = await readQueryResults({
cliServer: this.cliServer,
bqrsPath: queryResult.outputDir.bqrsPath,
});
if (!bqrsChunk) {
return;
}
const bqrsChunk = await readQueryResults({
cliServer: this.cliServer,
bqrsPath: queryResult.outputDir.bqrsPath,
});
if (!bqrsChunk) {
await this.clearProgress();
return;
}
progress({
message: "Finalizing results",
step: 1450,
maxStep: 1500,
});
await this.showProgress({
message: "Finalizing results",
step: 1450,
maxStep: 1500,
});
const externalApiUsages = decodeBqrsToExternalApiUsages(bqrsChunk);
const externalApiUsages = decodeBqrsToExternalApiUsages(bqrsChunk);
await this.postMessage({
t: "setExternalApiUsages",
externalApiUsages,
});
await this.clearProgress();
} catch (err) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
this.app.telemetry,
redactableError(
asError(err),
)`Failed to load external API usages: ${getErrorMessage(err)}`,
);
}
await this.postMessage({
t: "setExternalApiUsages",
externalApiUsages,
});
} catch (err) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
this.app.telemetry,
redactableError(
asError(err),
)`Failed to load external API usages: ${getErrorMessage(err)}`,
);
}
},
{ cancellable: false },
);
}
protected async generateModeledMethods(): Promise<void> {
const tokenSource = new CancellationTokenSource();
await withProgress(
async (progress) => {
const tokenSource = new CancellationTokenSource();
let addedDatabase: DatabaseItem | undefined;
let addedDatabase: DatabaseItem | undefined;
// In application mode, we need the database of a specific library to generate
// the modeled methods. In framework mode, we'll use the current database.
if (this.mode === Mode.Application) {
addedDatabase = await this.promptImportDatabase((update) =>
this.showProgress(update),
);
if (!addedDatabase) {
return;
}
}
await this.showProgress({
step: 0,
maxStep: 4000,
message: "Generating modeled methods for library",
});
try {
await generateFlowModel({
cliServer: this.cliServer,
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: addedDatabase ?? this.databaseItem,
onResults: async (modeledMethods) => {
const modeledMethodsByName: Record<string, ModeledMethod> = {};
for (const modeledMethod of modeledMethods) {
modeledMethodsByName[modeledMethod.signature] = modeledMethod;
// In application mode, we need the database of a specific library to generate
// the modeled methods. In framework mode, we'll use the current database.
if (this.mode === Mode.Application) {
addedDatabase = await this.promptImportDatabase(progress);
if (!addedDatabase) {
return;
}
}
await this.postMessage({
t: "addModeledMethods",
modeledMethods: modeledMethodsByName,
progress({
step: 0,
maxStep: 4000,
message: "Generating modeled methods for library",
});
try {
await generateFlowModel({
cliServer: this.cliServer,
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: addedDatabase ?? this.databaseItem,
onResults: async (modeledMethods) => {
const modeledMethodsByName: Record<string, ModeledMethod> = {};
for (const modeledMethod of modeledMethods) {
modeledMethodsByName[modeledMethod.signature] = modeledMethod;
}
await this.postMessage({
t: "addModeledMethods",
modeledMethods: modeledMethodsByName,
});
},
progress,
token: tokenSource.token,
});
},
progress: (update) => this.showProgress(update),
token: tokenSource.token,
});
} catch (e: unknown) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
this.app.telemetry,
redactableError(
asError(e),
)`Failed to generate flow model: ${getErrorMessage(e)}`,
);
}
} catch (e: unknown) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
this.app.telemetry,
redactableError(
asError(e),
)`Failed to generate flow model: ${getErrorMessage(e)}`,
);
}
if (addedDatabase) {
// After the flow model has been generated, we can remove the temporary database
// which we used for generating the flow model.
await this.showProgress({
step: 3900,
maxStep: 4000,
message: "Removing temporary database",
});
await this.databaseManager.removeDatabaseItem(addedDatabase);
}
await this.clearProgress();
if (addedDatabase) {
// After the flow model has been generated, we can remove the temporary database
// which we used for generating the flow model.
progress({
step: 3900,
maxStep: 4000,
message: "Removing temporary database",
});
await this.databaseManager.removeDatabaseItem(addedDatabase);
}
},
{ cancellable: false },
);
}
private async generateModeledMethodsFromLlm(
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
): Promise<void> {
const maxStep = 3000;
await withProgress(
async (progress) => {
const maxStep = 3000;
await this.showProgress({
step: 0,
maxStep,
message: "Retrieving usages",
});
progress({
step: 0,
maxStep,
message: "Retrieving usages",
});
const usages = await getAutoModelUsages({
cliServer: this.cliServer,
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: this.databaseItem,
progress: (update) => this.showProgress(update, maxStep),
});
const usages = await getAutoModelUsages({
cliServer: this.cliServer,
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: this.databaseItem,
progress: (update) => progress({ ...update, maxStep }),
});
await this.showProgress({
step: 1800,
maxStep,
message: "Creating request",
});
progress({
step: 1800,
maxStep,
message: "Creating request",
});
const request = createAutoModelRequest(
this.databaseItem.language,
externalApiUsages,
modeledMethods,
usages,
this.mode,
const request = createAutoModelRequest(
this.databaseItem.language,
externalApiUsages,
modeledMethods,
usages,
this.mode,
);
progress({
step: 2000,
maxStep,
message: "Sending request",
});
const response = await this.callAutoModelApi(request);
if (!response) {
return;
}
progress({
step: 2500,
maxStep,
message: "Parsing response",
});
const predictedModeledMethods = parsePredictedClassifications(
response.predicted || [],
);
progress({
step: 2800,
maxStep,
message: "Applying results",
});
await this.postMessage({
t: "addModeledMethods",
modeledMethods: predictedModeledMethods,
});
},
{ cancellable: false },
);
await this.showProgress({
step: 2000,
maxStep,
message: "Sending request",
});
const response = await this.callAutoModelApi(request);
if (!response) {
return;
}
await this.showProgress({
step: 2500,
maxStep,
message: "Parsing response",
});
const predictedModeledMethods = parsePredictedClassifications(
response.predicted || [],
);
await this.showProgress({
step: 2800,
maxStep,
message: "Applying results",
});
await this.postMessage({
t: "addModeledMethods",
modeledMethods: predictedModeledMethods,
});
await this.clearProgress();
}
private async modelDependency(): Promise<void> {
@@ -482,46 +480,12 @@ export class DataExtensionsEditorView extends AbstractWebview<
return addedDatabase;
}
/*
* Progress in this class is a bit weird. Most of the progress is based on running the query.
* Query progress is always between 0 and 1000. However, we still have some steps that need
* to be done after the query has finished. Therefore, the maximum step is 1500. This captures
* that there's 1000 steps of the query progress since that takes the most time, and then
* an additional 500 steps for the rest of the work. The progress doesn't need to be 100%
* accurate, so this is just a rough estimate.
*
* For generating the modeled methods for an external library, the max step is 4000. This is
* based on the following steps:
* - 1000 for the summary model
* - 1000 for the sink model
* - 1000 for the source model
* - 1000 for the neutral model
*/
private async showProgress(update: ProgressUpdate, maxStep?: number) {
await this.postMessage({
t: "showProgress",
step: update.step,
maxStep: maxStep ?? update.maxStep,
message: update.message,
});
}
private async clearProgress() {
await this.showProgress({
step: 0,
maxStep: 0,
message: "",
});
}
private async callAutoModelApi(
request: ModelRequest,
): Promise<ModelResponse | null> {
try {
return await autoModel(this.app.credentials, request);
} catch (e) {
await this.clearProgress();
if (e instanceof RequestError && e.status === 429) {
void showAndLogExceptionWithTelemetry(
this.app.logger,

2
extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts
View File

@@ -14,7 +14,7 @@ import { QueryResultType } from "../query-server/new-messages";
import { file } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump } from "js-yaml";
import { qlpackOfDatabase } from "../language-support";
import { qlpackOfDatabase } from "../local-queries";
import { telemetryListener } from "../common/vscode/telemetry";
type FlowModelOptions = {

7
extensions/ql-vscode/src/language-support/contextual/location-finder.ts
View File

@@ -12,16 +12,13 @@ import { CodeQLCliServer } from "../../codeql-cli/cli";
import { DatabaseManager, DatabaseItem } from "../../databases/local-databases";
import { ProgressCallback } from "../../common/vscode/progress";
import { KeyType } from "./key-type";
import {
qlpackOfDatabase,
resolveQueries,
runContextualQuery,
} from "./query-resolver";
import { resolveQueries, runContextualQuery } from "./query-resolver";
import { CancellationToken, LocationLink, Uri } from "vscode";
import { QueryOutputDir } from "../../run-queries-shared";
import { QueryRunner } from "../../query-server";
import { QueryResultType } from "../../query-server/new-messages";
import { fileRangeFromURI } from "./file-range-from-uri";
import { qlpackOfDatabase } from "../../local-queries";
export const SELECT_QUERY_NAME = "#select";
export const SELECTED_SOURCE_FILE = "selectedSourceFile";

164
extensions/ql-vscode/src/language-support/contextual/query-resolver.ts
View File

@@ -1,14 +1,5 @@
import { writeFile, promises } from "fs-extra";
import { dump } from "js-yaml";
import { file } from "tmp-promise";
import { basename, dirname, resolve } from "path";
import { getOnDiskWorkspaceFolders } from "../../common/vscode/workspace-folders";
import {
getPrimaryDbscheme,
getQlPackForDbscheme,
QlPacksForLanguage,
} from "../../databases/qlpack";
import { QlPacksForLanguage } from "../../databases/qlpack";
import {
KeyType,
kindOfKeyType,
@@ -17,154 +8,22 @@ import {
} from "./key-type";
import { CodeQLCliServer } from "../../codeql-cli/cli";
import { DatabaseItem } from "../../databases/local-databases";
import { resolveQueries as resolveLocalQueries } from "../../local-queries/query-resolver";
import { extLogger } from "../../common/logging/vscode";
import {
showAndLogExceptionWithTelemetry,
TeeLogger,
} from "../../common/logging";
import { TeeLogger } from "../../common/logging";
import { CancellationToken } from "vscode";
import { ProgressCallback } from "../../common/vscode/progress";
import { CoreCompletedQuery, QueryRunner } from "../../query-server";
import { redactableError } from "../../common/errors";
import { QLPACK_FILENAMES } from "../../common/ql";
import { telemetryListener } from "../../common/vscode/telemetry";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
db: Pick<DatabaseItem, "contents">,
): Promise<QlPacksForLanguage> {
if (db.contents === undefined) {
throw new Error("Database is invalid and cannot infer QLPack.");
}
const datasetPath = db.contents.datasetUri.fsPath;
const dbscheme = await getPrimaryDbscheme(datasetPath);
return await getQlPackForDbscheme(cli, dbscheme);
}
/**
* Finds the contextual queries with the specified key in a list of CodeQL packs.
*
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param keyType The contextual query key of the query to search for.
* @returns The found queries from the first pack in which any matching queries were found.
*/
async function resolveQueriesFromPacks(
cli: CodeQLCliServer,
qlpacks: string[],
keyType: KeyType,
): Promise<string[]> {
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
for (const qlpack of qlpacks) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: {
kind: kindOfKeyType(keyType),
"tags contain": tagOfKeyType(keyType),
},
});
}
await writeFile(suiteFile, dump(suiteYaml), "utf8");
const queries = await cli.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
return queries;
}
import { createLockFileForStandardQuery } from "../../local-queries/standard-queries";
export async function resolveQueries(
cli: CodeQLCliServer,
qlpacks: QlPacksForLanguage,
keyType: KeyType,
): Promise<string[]> {
const packsToSearch: string[] = [];
// The CLI can handle both library packs and query packs, so search both packs in order.
packsToSearch.push(qlpacks.dbschemePack);
if (qlpacks.queryPack !== undefined) {
packsToSearch.push(qlpacks.queryPack);
}
const queries = await resolveQueriesFromPacks(cli, packsToSearch, keyType);
if (queries.length > 0) {
return queries;
}
// No queries found. Determine the correct error message for the various scenarios.
const keyTypeName = nameOfKeyType(keyType);
const keyTypeTag = tagOfKeyType(keyType);
const joinedPacksToSearch = packsToSearch.join(", ");
const error = redactableError`No ${keyTypeName} queries (tagged "${keyTypeTag}") could be found in the \
current library path (tried searching the following packs: ${joinedPacksToSearch}). \
Try upgrading the CodeQL libraries. If that doesn't work, then ${keyTypeName} queries are not yet available \
for this language.`;
void showAndLogExceptionWithTelemetry(extLogger, telemetryListener, error);
throw error;
}
async function resolveContextualQuery(
cli: CodeQLCliServer,
query: string,
): Promise<{ packPath: string; createdTempLockFile: boolean }> {
// Contextual queries now live within the standard library packs.
// This simplifies distribution (you don't need the standard query pack to use the AST viewer),
// but if the library pack doesn't have a lockfile, we won't be able to find
// other pack dependencies of the library pack.
// Work out the enclosing pack.
const packContents = await cli.packPacklist(query, false);
const packFilePath = packContents.find((p) =>
QLPACK_FILENAMES.includes(basename(p)),
);
if (packFilePath === undefined) {
// Should not happen; we already resolved this query.
throw new Error(
`Could not find a CodeQL pack file for the pack enclosing the contextual query ${query}`,
);
}
const packPath = dirname(packFilePath);
const lockFilePath = packContents.find((p) =>
["codeql-pack.lock.yml", "qlpack.lock.yml"].includes(basename(p)),
);
let createdTempLockFile = false;
if (!lockFilePath) {
// No lock file, likely because this library pack is in the package cache.
// Create a lock file so that we can resolve dependencies and library path
// for the contextual query.
void extLogger.log(
`Library pack ${packPath} is missing a lock file; creating a temporary lock file`,
);
await cli.packResolveDependencies(packPath);
createdTempLockFile = true;
// Clear CLI server pack cache before installing dependencies,
// so that it picks up the new lock file, not the previously cached pack.
void extLogger.log("Clearing the CodeQL CLI server's pack cache");
await cli.clearCache();
// Install dependencies.
void extLogger.log(
`Installing package dependencies for library pack ${packPath}`,
);
await cli.packInstall(packPath);
}
return { packPath, createdTempLockFile };
}
async function removeTemporaryLockFile(packPath: string) {
const tempLockFilePath = resolve(packPath, "codeql-pack.lock.yml");
void extLogger.log(
`Deleting temporary package lock file at ${tempLockFilePath}`,
);
// It's fine if the file doesn't exist.
await promises.rm(resolve(packPath, "codeql-pack.lock.yml"), {
force: true,
return resolveLocalQueries(cli, qlpacks, nameOfKeyType(keyType), {
kind: kindOfKeyType(keyType),
"tags contain": [tagOfKeyType(keyType)],
});
}
@@ -178,10 +37,7 @@ export async function runContextualQuery(
token: CancellationToken,
templates: Record<string, string>,
): Promise<CoreCompletedQuery> {
const { packPath, createdTempLockFile } = await resolveContextualQuery(
cli,
query,
);
const { cleanup } = await createLockFileForStandardQuery(cli, query);
const queryRun = qs.createQueryRun(
db.databaseUri.fsPath,
{ queryPath: query, quickEvalPosition: undefined },
@@ -200,8 +56,6 @@ export async function runContextualQuery(
token,
new TeeLogger(qs.logger, queryRun.outputDir.logPath),
);
if (createdTempLockFile) {
await removeTemporaryLockFile(packPath);
}
await cleanup?.();
return results;
}

7
extensions/ql-vscode/src/language-support/contextual/template-provider.ts
View File

@@ -27,11 +27,7 @@ import {
SELECTED_SOURCE_LINE,
SELECTED_SOURCE_COLUMN,
} from "./location-finder";
import {
qlpackOfDatabase,
resolveQueries,
runContextualQuery,
} from "./query-resolver";
import { resolveQueries, runContextualQuery } from "./query-resolver";
import {
isCanary,
NO_CACHE_AST_VIEWER,
@@ -39,6 +35,7 @@ import {
} from "../../config";
import { CoreCompletedQuery, QueryRunner } from "../../query-server";
import { AstBuilder } from "../ast-viewer/ast-builder";
import { qlpackOfDatabase } from "../../local-queries";
/**
* Runs templated CodeQL queries to find definitions in

1
extensions/ql-vscode/src/local-queries/index.ts
View File

@@ -1,5 +1,6 @@
export * from "./local-queries";
export * from "./local-query-run";
export * from "./query-resolver";
export * from "./quick-eval-code-lens-provider";
export * from "./quick-query";
export * from "./results-view";

131
extensions/ql-vscode/src/local-queries/query-resolver.ts Normal file
View File

@@ -0,0 +1,131 @@
import { CodeQLCliServer } from "../codeql-cli/cli";
import { DatabaseItem } from "../databases/local-databases";
import {
getPrimaryDbscheme,
getQlPackForDbscheme,
QlPacksForLanguage,
} from "../databases/qlpack";
import { file } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump } from "js-yaml";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { redactableError } from "../common/errors";
import { showAndLogExceptionWithTelemetry } from "../common/logging";
import { extLogger } from "../common/logging/vscode";
import { telemetryListener } from "../common/vscode/telemetry";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
db: Pick<DatabaseItem, "contents">,
): Promise<QlPacksForLanguage> {
if (db.contents === undefined) {
throw new Error("Database is invalid and cannot infer QLPack.");
}
const datasetPath = db.contents.datasetUri.fsPath;
const dbscheme = await getPrimaryDbscheme(datasetPath);
return await getQlPackForDbscheme(cli, dbscheme);
}
export interface QueryConstraints {
kind?: string;
"tags contain"?: string[];
"tags contain all"?: string[];
}
/**
* Finds the queries with the specified kind and tags in a list of CodeQL packs.
*
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param constraints Constraints on the queries to search for.
* @returns The found queries from the first pack in which any matching queries were found.
*/
async function resolveQueriesFromPacks(
cli: CodeQLCliServer,
qlpacks: string[],
constraints: QueryConstraints,
): Promise<string[]> {
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
for (const qlpack of qlpacks) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: constraints,
});
}
await writeFile(
suiteFile,
dump(suiteYaml, {
noRefs: true, // CodeQL doesn't really support refs
}),
"utf8",
);
return await cli.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
}
/**
* Finds the queries with the specified kind and tags in a QLPack.
*
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param name The name of the query to use in error messages.
* @param constraints Constraints on the queries to search for.
* @returns The found queries from the first pack in which any matching queries were found.
*/
export async function resolveQueries(
cli: CodeQLCliServer,
qlpacks: QlPacksForLanguage,
name: string,
constraints: QueryConstraints,
): Promise<string[]> {
const packsToSearch: string[] = [];
// The CLI can handle both library packs and query packs, so search both packs in order.
packsToSearch.push(qlpacks.dbschemePack);
if (qlpacks.queryPack !== undefined) {
packsToSearch.push(qlpacks.queryPack);
}
const queries = await resolveQueriesFromPacks(
cli,
packsToSearch,
constraints,
);
if (queries.length > 0) {
return queries;
}
// No queries found. Determine the correct error message for the various scenarios.
const humanConstraints = [];
if (constraints.kind !== undefined) {
humanConstraints.push(`kind "${constraints.kind}"`);
}
if (constraints["tags contain"] !== undefined) {
humanConstraints.push(`tagged "${constraints["tags contain"].join(" ")}"`);
}
if (constraints["tags contain all"] !== undefined) {
humanConstraints.push(
`tagged all of "${constraints["tags contain all"].join(" ")}"`,
);
}
const joinedPacksToSearch = packsToSearch.join(", ");
const error = redactableError`No ${name} queries (${humanConstraints.join(
", ",
)}) could be found in the \
current library path (tried searching the following packs: ${joinedPacksToSearch}). \
Try upgrading the CodeQL libraries. If that doesn't work, then ${name} queries are not yet available \
for this language.`;
void showAndLogExceptionWithTelemetry(extLogger, telemetryListener, error);
throw error;
}

77
extensions/ql-vscode/src/local-queries/standard-queries.ts Normal file
View File

@@ -0,0 +1,77 @@
import { CodeQLCliServer } from "../codeql-cli/cli";
import { QLPACK_FILENAMES, QLPACK_LOCK_FILENAMES } from "../common/ql";
import { basename, dirname, resolve } from "path";
import { extLogger } from "../common/logging/vscode";
import { promises } from "fs-extra";
import { BaseLogger } from "../common/logging";
type LockFileForStandardQueryResult = {
cleanup?: () => Promise<void>;
};
/**
* Create a temporary query suite for a given query living within the standard library packs.
*
* This will create a lock file so the CLI can run the query without having the ql submodule.
*/
export async function createLockFileForStandardQuery(
cli: CodeQLCliServer,
queryPath: string,
logger: BaseLogger = extLogger,
): Promise<LockFileForStandardQueryResult> {
// These queries live within the standard library packs.
// This simplifies distribution (you don't need the standard query pack to use the AST viewer),
// but if the library pack doesn't have a lockfile, we won't be able to find
// other pack dependencies of the library pack.
// Work out the enclosing pack.
const packContents = await cli.packPacklist(queryPath, false);
const packFilePath = packContents.find((p) =>
QLPACK_FILENAMES.includes(basename(p)),
);
if (packFilePath === undefined) {
// Should not happen; we already resolved this query.
throw new Error(
`Could not find a CodeQL pack file for the pack enclosing the contextual query ${queryPath}`,
);
}
const packPath = dirname(packFilePath);
const lockFilePath = packContents.find((p) =>
QLPACK_LOCK_FILENAMES.includes(basename(p)),
);
let cleanup: (() => Promise<void>) | undefined = undefined;
if (!lockFilePath) {
// No lock file, likely because this library pack is in the package cache.
// Create a lock file so that we can resolve dependencies and library path
// for the contextual query.
void logger.log(
`Library pack ${packPath} is missing a lock file; creating a temporary lock file`,
);
await cli.packResolveDependencies(packPath);
cleanup = async () => {
const tempLockFilePath = resolve(packPath, "codeql-pack.lock.yml");
void logger.log(
`Deleting temporary package lock file at ${tempLockFilePath}`,
);
// It's fine if the file doesn't exist.
await promises.rm(resolve(packPath, "codeql-pack.lock.yml"), {
force: true,
});
};
// Clear CLI server pack cache before installing dependencies,
// so that it picks up the new lock file, not the previously cached pack.
void logger.log("Clearing the CodeQL CLI server's pack cache");
await cli.clearCache();
// Install dependencies.
void logger.log(
`Installing package dependencies for library pack ${packPath}`,
);
await cli.packInstall(packPath);
}
return { cleanup };
}

152
extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx
View File

@@ -1,9 +1,6 @@
import * as React from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import {
ShowProgressMessage,
ToDataExtensionsEditorMessage,
} from "../../common/interface-types";
import { ToDataExtensionsEditorMessage } from "../../common/interface-types";
import { VSCodeButton } from "@vscode/webview-ui-toolkit/react";
import styled from "styled-components";
import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usage";
@@ -46,17 +43,6 @@ const ButtonsContainer = styled.div`
margin-bottom: 1rem;
`;
type ProgressBarProps = {
completion: number;
};
const ProgressBar = styled.div<ProgressBarProps>`
height: 10px;
width: ${(props) => props.completion * 100}%;
background-color: var(--vscode-progressBar-background);
`;
type Props = {
initialViewState?: DataExtensionEditorViewState;
initialExternalApiUsages?: ExternalApiUsage[];
@@ -82,11 +68,6 @@ export function DataExtensionsEditor({
const [modeledMethods, setModeledMethods] = useState<
Record<string, ModeledMethod>
>(initialModeledMethods);
const [progress, setProgress] = useState<Omit<ShowProgressMessage, "t">>({
step: 0,
maxStep: 0,
message: "",
});
useEffect(() => {
const listener = (evt: MessageEvent) => {
@@ -99,9 +80,6 @@ export function DataExtensionsEditor({
case "setExternalApiUsages":
setExternalApiUsages(msg.externalApiUsages);
break;
case "showProgress":
setProgress(msg);
break;
case "loadModeledMethods":
setModeledMethods((oldModeledMethods) => {
return {
@@ -244,89 +222,71 @@ export function DataExtensionsEditor({
});
}, [viewState?.mode]);
if (viewState === undefined) {
if (viewState === undefined || externalApiUsages.length === 0) {
return <LoadingContainer>Loading...</LoadingContainer>;
}
return (
<DataExtensionsEditorContainer>
{progress.maxStep > 0 && (
<p>
<ProgressBar completion={progress.step / progress.maxStep} />{" "}
{progress.message}
</p>
)}
{externalApiUsages.length > 0 && (
<>
<ViewTitle>
{getLanguageDisplayName(viewState.extensionPack.language)}
</ViewTitle>
<DetailsContainer>
<LinkIconButton onClick={onOpenExtensionPackClick}>
<span slot="start" className="codicon codicon-package"></span>
{viewState.extensionPack.name}
</LinkIconButton>
<ViewTitle>
{getLanguageDisplayName(viewState.extensionPack.language)}
</ViewTitle>
<DetailsContainer>
<LinkIconButton onClick={onOpenExtensionPackClick}>
<span slot="start" className="codicon codicon-package"></span>
{viewState.extensionPack.name}
</LinkIconButton>
<div>{percentFormatter.format(modeledPercentage / 100)} modeled</div>
<div>
{percentFormatter.format(unModeledPercentage / 100)} unmodeled
</div>
{viewState.enableFrameworkMode && (
<>
<div>
{percentFormatter.format(modeledPercentage / 100)} modeled
Mode:{" "}
{viewState.mode === Mode.Framework ? "Framework" : "Application"}
</div>
<div>
{percentFormatter.format(unModeledPercentage / 100)} unmodeled
<LinkIconButton onClick={onSwitchModeClick}>
<span slot="start" className="codicon codicon-library"></span>
Switch mode
</LinkIconButton>
</div>
{viewState.enableFrameworkMode && (
<>
<div>
Mode:{" "}
{viewState.mode === Mode.Framework
? "Framework"
: "Application"}
</div>
<div>
<LinkIconButton onClick={onSwitchModeClick}>
<span
slot="start"
className="codicon codicon-library"
></span>
Switch mode
</LinkIconButton>
</div>
</>
)}
</DetailsContainer>
</>
)}
</DetailsContainer>
<EditorContainer>
<ButtonsContainer>
<VSCodeButton
onClick={onSaveAllClick}
disabled={modifiedSignatures.size === 0}
>
Save all
</VSCodeButton>
{viewState.enableFrameworkMode && (
<VSCodeButton appearance="secondary" onClick={onRefreshClick}>
Refresh
</VSCodeButton>
)}
{viewState.mode === Mode.Framework && (
<VSCodeButton onClick={onGenerateFromSourceClick}>
Generate
</VSCodeButton>
)}
</ButtonsContainer>
<ModeledMethodsList
externalApiUsages={externalApiUsages}
modeledMethods={modeledMethods}
modifiedSignatures={modifiedSignatures}
viewState={viewState}
onChange={onChange}
onSaveModelClick={onSaveModelClick}
onGenerateFromLlmClick={onGenerateFromLlmClick}
onGenerateFromSourceClick={onGenerateFromSourceClick}
onModelDependencyClick={onModelDependencyClick}
/>
</EditorContainer>
</>
)}
<EditorContainer>
<ButtonsContainer>
<VSCodeButton
onClick={onSaveAllClick}
disabled={modifiedSignatures.size === 0}
>
Save all
</VSCodeButton>
{viewState.enableFrameworkMode && (
<VSCodeButton appearance="secondary" onClick={onRefreshClick}>
Refresh
</VSCodeButton>
)}
{viewState.mode === Mode.Framework && (
<VSCodeButton onClick={onGenerateFromSourceClick}>
Generate
</VSCodeButton>
)}
</ButtonsContainer>
<ModeledMethodsList
externalApiUsages={externalApiUsages}
modeledMethods={modeledMethods}
modifiedSignatures={modifiedSignatures}
viewState={viewState}
onChange={onChange}
onSaveModelClick={onSaveModelClick}
onGenerateFromLlmClick={onGenerateFromLlmClick}
onGenerateFromSourceClick={onGenerateFromSourceClick}
onModelDependencyClick={onModelDependencyClick}
/>
</EditorContainer>
</DataExtensionsEditorContainer>
);
}

52
extensions/ql-vscode/test/vscode-tests/no-workspace/language-support/contextual/query-resolver.test.ts
View File

@@ -5,23 +5,11 @@ import { getErrorMessage } from "../../../../../src/common/helpers-pure";
import * as log from "../../../../../src/common/logging/notifications";
import * as workspaceFolders from "../../../../../src/common/vscode/workspace-folders";
import * as qlpack from "../../../../../src/databases/qlpack";
import {
KeyType,
qlpackOfDatabase,
resolveQueries,
} from "../../../../../src/language-support";
import { KeyType, resolveQueries } from "../../../../../src/language-support";
import { CodeQLCliServer } from "../../../../../src/codeql-cli/cli";
import { mockDatabaseItem, mockedObject } from "../../../utils/mocking.helpers";
import { mockedObject } from "../../../utils/mocking.helpers";
describe("queryResolver", () => {
let getQlPackForDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getQlPackForDbscheme
>;
let getPrimaryDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getPrimaryDbscheme
>;
const resolveQueriesInSuite = jest.fn();
const mockCli = mockedObject<CodeQLCliServer>({
@@ -29,16 +17,6 @@ describe("queryResolver", () => {
});
beforeEach(() => {
getQlPackForDbschemeSpy = jest
.spyOn(qlpack, "getQlPackForDbscheme")
.mockResolvedValue({
dbschemePack: "dbschemePack",
dbschemePackIsLibraryPack: false,
});
getPrimaryDbschemeSpy = jest
.spyOn(qlpack, "getPrimaryDbscheme")
.mockResolvedValue("primaryDbscheme");
jest
.spyOn(workspaceFolders, "getOnDiskWorkspaceFolders")
.mockReturnValue([]);
@@ -68,7 +46,7 @@ describe("queryResolver", () => {
queries: ".",
include: {
kind: "definitions",
"tags contain": "ide-contextual-queries/local-definitions",
"tags contain": ["ide-contextual-queries/local-definitions"],
},
},
]);
@@ -87,31 +65,9 @@ describe("queryResolver", () => {
expect(true).toBe(false);
} catch (e) {
expect(getErrorMessage(e)).toBe(
'No definitions queries (tagged "ide-contextual-queries/local-definitions") could be found in the current library path (tried searching the following packs: my-qlpack). Try upgrading the CodeQL libraries. If that doesn\'t work, then definitions queries are not yet available for this language.',
'No definitions queries (kind "definitions", tagged "ide-contextual-queries/local-definitions") could be found in the current library path (tried searching the following packs: my-qlpack). Try upgrading the CodeQL libraries. If that doesn\'t work, then definitions queries are not yet available for this language.',
);
}
});
});
describe("qlpackOfDatabase", () => {
it("should get the qlpack of a database", async () => {
getQlPackForDbschemeSpy.mockResolvedValue({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
const db = mockDatabaseItem({
contents: {
datasetUri: {
fsPath: "/path/to/database",
},
},
});
const result = await qlpackOfDatabase(mockCli, db);
expect(result).toEqual({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
expect(getPrimaryDbschemeSpy).toBeCalledWith("/path/to/database");
});
});
});

119
extensions/ql-vscode/test/vscode-tests/no-workspace/local-queries/query-resolver.test.ts Normal file
View File

@@ -0,0 +1,119 @@
import {
qlpackOfDatabase,
resolveQueries,
} from "../../../../src/local-queries";
import { mockDatabaseItem, mockedObject } from "../../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import * as qlpack from "../../../../src/databases/qlpack";
import * as workspaceFolders from "../../../../src/common/vscode/workspace-folders";
import * as log from "../../../../src/common/logging/notifications";
import { load } from "js-yaml";
import * as fs from "fs-extra";
describe("qlpackOfDatabase", () => {
let getQlPackForDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getQlPackForDbscheme
>;
let getPrimaryDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getPrimaryDbscheme
>;
const mockCli = mockedObject<CodeQLCliServer>({});
beforeEach(() => {
getQlPackForDbschemeSpy = jest
.spyOn(qlpack, "getQlPackForDbscheme")
.mockResolvedValue({
dbschemePack: "dbschemePack",
dbschemePackIsLibraryPack: false,
});
getPrimaryDbschemeSpy = jest
.spyOn(qlpack, "getPrimaryDbscheme")
.mockResolvedValue("primaryDbscheme");
});
it("should get the qlpack of a database", async () => {
getQlPackForDbschemeSpy.mockResolvedValue({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
const db = mockDatabaseItem({
contents: {
datasetUri: {
fsPath: "/path/to/database",
},
},
});
const result = await qlpackOfDatabase(mockCli, db);
expect(result).toEqual({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
expect(getPrimaryDbschemeSpy).toBeCalledWith("/path/to/database");
});
});
describe("resolveQueries", () => {
const resolveQueriesInSuite = jest.fn();
const mockCli = mockedObject<CodeQLCliServer>({
resolveQueriesInSuite,
});
beforeEach(() => {
jest
.spyOn(workspaceFolders, "getOnDiskWorkspaceFolders")
.mockReturnValue([]);
jest.spyOn(log, "showAndLogErrorMessage").mockResolvedValue(undefined);
});
it("should resolve a query", async () => {
resolveQueriesInSuite.mockReturnValue(["a", "b"]);
const result = await resolveQueries(
mockCli,
{ dbschemePack: "my-qlpack", dbschemePackIsLibraryPack: false },
"my query",
{
kind: "graph",
"tags contain": ["ide-contextual-queries/print-ast"],
},
);
expect(result).toEqual(["a", "b"]);
expect(resolveQueriesInSuite).toHaveBeenCalledWith(
expect.stringMatching(/\.qls$/),
[],
);
const fileName = resolveQueriesInSuite.mock.calls[0][0];
expect(load(await fs.readFile(fileName, "utf-8"))).toEqual([
{
from: "my-qlpack",
queries: ".",
include: {
kind: "graph",
"tags contain": ["ide-contextual-queries/print-ast"],
},
},
]);
});
it("should throw an error when there are no queries found", async () => {
resolveQueriesInSuite.mockReturnValue([]);
await expect(
resolveQueries(
mockCli,
{ dbschemePack: "my-qlpack", dbschemePackIsLibraryPack: false },
"my query",
{
kind: "graph",
"tags contain": ["ide-contextual-queries/print-ast"],
},
),
).rejects.toThrowError(
'No my query queries (kind "graph", tagged "ide-contextual-queries/print-ast") could be found in the current library path (tried searching the following packs: my-qlpack). Try upgrading the CodeQL libraries. If that doesn\'t work, then my query queries are not yet available for this language.',
);
});
});

113
extensions/ql-vscode/test/vscode-tests/no-workspace/local-queries/standard-queries.test.ts Normal file
View File

@@ -0,0 +1,113 @@
import { mockedObject } from "../../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import { dir, DirectoryResult } from "tmp-promise";
import { join } from "path";
import { createLockFileForStandardQuery } from "../../../../src/local-queries/standard-queries";
import { outputFile, pathExists } from "fs-extra";
describe("createLockFileForStandardQuery", () => {
let tmpDir: DirectoryResult;
let packPath: string;
let qlpackPath: string;
let queryPath: string;
const packPacklist = jest.fn();
const packResolveDependencies = jest.fn();
const clearCache = jest.fn();
const packInstall = jest.fn();
const mockCli = mockedObject<CodeQLCliServer>({
packPacklist,
packResolveDependencies,
clearCache,
packInstall,
});
beforeEach(async () => {
tmpDir = await dir({
unsafeCleanup: true,
});
packPath = join(tmpDir.path, "a", "b");
qlpackPath = join(packPath, "qlpack.yml");
queryPath = join(packPath, "d", "e", "query.ql");
packPacklist.mockResolvedValue([qlpackPath, queryPath]);
});
afterEach(async () => {
await tmpDir.cleanup();
});
describe("when the lock file exists", () => {
let lockfilePath: string;
beforeEach(async () => {
lockfilePath = join(packPath, "qlpack.lock.yml");
packPacklist.mockResolvedValue([qlpackPath, lockfilePath, queryPath]);
});
it("does not resolve or install dependencies", async () => {
expect(await createLockFileForStandardQuery(mockCli, queryPath)).toEqual({
cleanup: undefined,
});
expect(packResolveDependencies).not.toHaveBeenCalled();
expect(clearCache).not.toHaveBeenCalled();
expect(packInstall).not.toHaveBeenCalled();
});
it("does not resolve or install dependencies with a codeql-pack.lock.yml", async () => {
lockfilePath = join(packPath, "codeql-pack.lock.yml");
packPacklist.mockResolvedValue([qlpackPath, lockfilePath, queryPath]);
expect(await createLockFileForStandardQuery(mockCli, queryPath)).toEqual({
cleanup: undefined,
});
expect(packResolveDependencies).not.toHaveBeenCalled();
expect(clearCache).not.toHaveBeenCalled();
expect(packInstall).not.toHaveBeenCalled();
});
});
describe("when the lock file does not exist", () => {
it("resolves and installs dependencies", async () => {
expect(await createLockFileForStandardQuery(mockCli, queryPath)).toEqual({
cleanup: expect.any(Function),
});
expect(packResolveDependencies).toHaveBeenCalledWith(packPath);
expect(clearCache).toHaveBeenCalledWith();
expect(packInstall).toHaveBeenCalledWith(packPath);
});
it("cleans up the lock file using the cleanup function", async () => {
const { cleanup } = await createLockFileForStandardQuery(
mockCli,
queryPath,
);
expect(cleanup).not.toBeUndefined();
const lockfilePath = join(packPath, "codeql-pack.lock.yml");
await outputFile(lockfilePath, "lock file contents");
await cleanup?.();
expect(await pathExists(lockfilePath)).toBe(false);
});
it("does not fail when cleaning up a non-existing lock file", async () => {
const { cleanup } = await createLockFileForStandardQuery(
mockCli,
queryPath,
);
expect(cleanup).not.toBeUndefined();
await cleanup?.();
});
});
});