mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into robertbrignull/data_new_header

Browse Source
This commit is contained in:
Robert
2023-07-26 10:29:22 +01:00
parent 3e49d05ef9 e877695a14
commit af50d90bcb
23 changed files with 707 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/test-plan.md
View File

@@ -318,7 +318,6 @@ This requires running a MRVA query and seeing the results view.
1. Alphabetically
2. By number of results
3. By popularity
4. By most recent commit
9. Can filter repos
10. Shows correct statistics
1. Total number of results

1
extensions/ql-vscode/CHANGELOG.md
View File

@@ -2,6 +2,7 @@
## [UNRELEASED]
- Remove "last updated" information and sorting from variant analysis results view. [#2637](https://github.com/github/vscode-codeql/pull/2637)
- Links to code on GitHub now include column numbers as well as line numbers. [#2406](https://github.com/github/vscode-codeql/pull/2406)
- No longer highlight trailing commas for jump to definition. [#2615](https://github.com/github/vscode-codeql/pull/2615)

2
extensions/ql-vscode/package.json
View File

@@ -350,13 +350,11 @@
"enum": [
"alphabetically",
"popularity",
"mostRecentCommit",
"numberOfResults"
],
"enumDescriptions": [
"Sort repositories alphabetically in the results view.",
"Sort repositories by popularity in the results view.",
"Sort repositories by most recent commit in the results view.",
"Sort repositories by number of results in the results view."
],
"description": "The default sorting order for repositories in the variant analysis results view."

8
extensions/ql-vscode/src/common/interface-types.ts
View File

@@ -500,13 +500,6 @@ interface SetExternalApiUsagesMessage {
externalApiUsages: ExternalApiUsage[];
}
export interface ShowProgressMessage {
t: "showProgress";
step: number;
maxStep: number;
message: string;
}
interface LoadModeledMethodsMessage {
t: "loadModeledMethods";
modeledMethods: Record<string, ModeledMethod>;
@@ -562,7 +555,6 @@ interface ModelDependencyMessage {
export type ToDataExtensionsEditorMessage =
| SetExtensionPackStateMessage
| SetExternalApiUsagesMessage
| ShowProgressMessage
| LoadModeledMethodsMessage
| AddModeledMethodsMessage;

96
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
View File

@@ -14,11 +14,7 @@ import {
FromDataExtensionsEditorMessage,
ToDataExtensionsEditorMessage,
} from "../common/interface-types";
import {
ProgressCallback,
ProgressUpdate,
withProgress,
} from "../common/vscode/progress";
import { ProgressCallback, withProgress } from "../common/vscode/progress";
import { QueryRunner } from "../query-server";
import {
showAndLogExceptionWithTelemetry,
@@ -232,9 +228,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
}
protected async loadExternalApiUsages(): Promise<void> {
const cancellationTokenSource = new CancellationTokenSource();
await withProgress(
async (progress) => {
try {
const cancellationTokenSource = new CancellationTokenSource();
const queryResult = await runQuery(
this.mode === Mode.Framework
? "frameworkModeQuery"
@@ -244,18 +241,15 @@ export class DataExtensionsEditorView extends AbstractWebview<
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
queryStorageDir: this.queryStorageDir,
progress: (progressUpdate: ProgressUpdate) => {
void this.showProgress(progressUpdate, 1500);
},
progress: (update) => progress({ ...update, maxStep: 1500 }),
token: cancellationTokenSource.token,
},
);
if (!queryResult) {
await this.clearProgress();
return;
}
await this.showProgress({
progress({
message: "Decoding results",
step: 1100,
maxStep: 1500,
@@ -266,11 +260,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
bqrsPath: queryResult.outputDir.bqrsPath,
});
if (!bqrsChunk) {
await this.clearProgress();
return;
}
await this.showProgress({
progress({
message: "Finalizing results",
step: 1450,
maxStep: 1500,
@@ -282,8 +275,6 @@ export class DataExtensionsEditorView extends AbstractWebview<
t: "setExternalApiUsages",
externalApiUsages,
});
await this.clearProgress();
} catch (err) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
@@ -293,9 +284,14 @@ export class DataExtensionsEditorView extends AbstractWebview<
)`Failed to load external API usages: ${getErrorMessage(err)}`,
);
}
},
{ cancellable: false },
);
}
protected async generateModeledMethods(): Promise<void> {
await withProgress(
async (progress) => {
const tokenSource = new CancellationTokenSource();
let addedDatabase: DatabaseItem | undefined;
@@ -303,15 +299,13 @@ export class DataExtensionsEditorView extends AbstractWebview<
// In application mode, we need the database of a specific library to generate
// the modeled methods. In framework mode, we'll use the current database.
if (this.mode === Mode.Application) {
addedDatabase = await this.promptImportDatabase((update) =>
this.showProgress(update),
);
addedDatabase = await this.promptImportDatabase(progress);
if (!addedDatabase) {
return;
}
}
await this.showProgress({
progress({
step: 0,
maxStep: 4000,
message: "Generating modeled methods for library",
@@ -335,7 +329,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
modeledMethods: modeledMethodsByName,
});
},
progress: (update) => this.showProgress(update),
progress,
token: tokenSource.token,
});
} catch (e: unknown) {
@@ -351,24 +345,27 @@ export class DataExtensionsEditorView extends AbstractWebview<
if (addedDatabase) {
// After the flow model has been generated, we can remove the temporary database
// which we used for generating the flow model.
await this.showProgress({
progress({
step: 3900,
maxStep: 4000,
message: "Removing temporary database",
});
await this.databaseManager.removeDatabaseItem(addedDatabase);
}
await this.clearProgress();
},
{ cancellable: false },
);
}
private async generateModeledMethodsFromLlm(
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
): Promise<void> {
await withProgress(
async (progress) => {
const maxStep = 3000;
await this.showProgress({
progress({
step: 0,
maxStep,
message: "Retrieving usages",
@@ -379,10 +376,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: this.databaseItem,
progress: (update) => this.showProgress(update, maxStep),
progress: (update) => progress({ ...update, maxStep }),
});
await this.showProgress({
progress({
step: 1800,
maxStep,
message: "Creating request",
@@ -396,7 +393,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
this.mode,
);
await this.showProgress({
progress({
step: 2000,
maxStep,
message: "Sending request",
@@ -407,7 +404,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
return;
}
await this.showProgress({
progress({
step: 2500,
maxStep,
message: "Parsing response",
@@ -417,7 +414,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
response.predicted || [],
);
await this.showProgress({
progress({
step: 2800,
maxStep,
message: "Applying results",
@@ -427,8 +424,9 @@ export class DataExtensionsEditorView extends AbstractWebview<
t: "addModeledMethods",
modeledMethods: predictedModeledMethods,
});
await this.clearProgress();
},
{ cancellable: false },
);
}
private async modelDependency(): Promise<void> {
@@ -489,46 +487,12 @@ export class DataExtensionsEditorView extends AbstractWebview<
return addedDatabase;
}
/*
* Progress in this class is a bit weird. Most of the progress is based on running the query.
* Query progress is always between 0 and 1000. However, we still have some steps that need
* to be done after the query has finished. Therefore, the maximum step is 1500. This captures
* that there's 1000 steps of the query progress since that takes the most time, and then
* an additional 500 steps for the rest of the work. The progress doesn't need to be 100%
* accurate, so this is just a rough estimate.
*
* For generating the modeled methods for an external library, the max step is 4000. This is
* based on the following steps:
* - 1000 for the summary model
* - 1000 for the sink model
* - 1000 for the source model
* - 1000 for the neutral model
*/
private async showProgress(update: ProgressUpdate, maxStep?: number) {
await this.postMessage({
t: "showProgress",
step: update.step,
maxStep: maxStep ?? update.maxStep,
message: update.message,
});
}
private async clearProgress() {
await this.showProgress({
step: 0,
maxStep: 0,
message: "",
});
}
private async callAutoModelApi(
request: ModelRequest,
): Promise<ModelResponse | null> {
try {
return await autoModel(this.app.credentials, request);
} catch (e) {
await this.clearProgress();
if (e instanceof RequestError && e.status === 429) {
void showAndLogExceptionWithTelemetry(
this.app.logger,

2
extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts
View File

@@ -14,7 +14,7 @@ import { QueryResultType } from "../query-server/new-messages";
import { file } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump } from "js-yaml";
import { qlpackOfDatabase } from "../language-support";
import { qlpackOfDatabase } from "../local-queries";
import { telemetryListener } from "../common/vscode/telemetry";
type FlowModelOptions = {

7
extensions/ql-vscode/src/language-support/contextual/location-finder.ts
View File

@@ -12,16 +12,13 @@ import { CodeQLCliServer } from "../../codeql-cli/cli";
import { DatabaseManager, DatabaseItem } from "../../databases/local-databases";
import { ProgressCallback } from "../../common/vscode/progress";
import { KeyType } from "./key-type";
import {
qlpackOfDatabase,
resolveQueries,
runContextualQuery,
} from "./query-resolver";
import { resolveQueries, runContextualQuery } from "./query-resolver";
import { CancellationToken, LocationLink, Uri } from "vscode";
import { QueryOutputDir } from "../../run-queries-shared";
import { QueryRunner } from "../../query-server";
import { QueryResultType } from "../../query-server/new-messages";
import { fileRangeFromURI } from "./file-range-from-uri";
import { qlpackOfDatabase } from "../../local-queries";
export const SELECT_QUERY_NAME = "#select";
export const SELECTED_SOURCE_FILE = "selectedSourceFile";

164
extensions/ql-vscode/src/language-support/contextual/query-resolver.ts
View File

@@ -1,14 +1,5 @@
import { writeFile, promises } from "fs-extra";
import { dump } from "js-yaml";
import { file } from "tmp-promise";
import { basename, dirname, resolve } from "path";
import { getOnDiskWorkspaceFolders } from "../../common/vscode/workspace-folders";
import {
getPrimaryDbscheme,
getQlPackForDbscheme,
QlPacksForLanguage,
} from "../../databases/qlpack";
import { QlPacksForLanguage } from "../../databases/qlpack";
import {
KeyType,
kindOfKeyType,
@@ -17,154 +8,22 @@ import {
} from "./key-type";
import { CodeQLCliServer } from "../../codeql-cli/cli";
import { DatabaseItem } from "../../databases/local-databases";
import { resolveQueries as resolveLocalQueries } from "../../local-queries/query-resolver";
import { extLogger } from "../../common/logging/vscode";
import {
showAndLogExceptionWithTelemetry,
TeeLogger,
} from "../../common/logging";
import { TeeLogger } from "../../common/logging";
import { CancellationToken } from "vscode";
import { ProgressCallback } from "../../common/vscode/progress";
import { CoreCompletedQuery, QueryRunner } from "../../query-server";
import { redactableError } from "../../common/errors";
import { QLPACK_FILENAMES } from "../../common/ql";
import { telemetryListener } from "../../common/vscode/telemetry";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
db: Pick<DatabaseItem, "contents">,
): Promise<QlPacksForLanguage> {
if (db.contents === undefined) {
throw new Error("Database is invalid and cannot infer QLPack.");
}
const datasetPath = db.contents.datasetUri.fsPath;
const dbscheme = await getPrimaryDbscheme(datasetPath);
return await getQlPackForDbscheme(cli, dbscheme);
}
/**
* Finds the contextual queries with the specified key in a list of CodeQL packs.
*
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param keyType The contextual query key of the query to search for.
* @returns The found queries from the first pack in which any matching queries were found.
*/
async function resolveQueriesFromPacks(
cli: CodeQLCliServer,
qlpacks: string[],
keyType: KeyType,
): Promise<string[]> {
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
for (const qlpack of qlpacks) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: {
kind: kindOfKeyType(keyType),
"tags contain": tagOfKeyType(keyType),
},
});
}
await writeFile(suiteFile, dump(suiteYaml), "utf8");
const queries = await cli.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
return queries;
}
import { createLockFileForStandardQuery } from "../../local-queries/standard-queries";
export async function resolveQueries(
cli: CodeQLCliServer,
qlpacks: QlPacksForLanguage,
keyType: KeyType,
): Promise<string[]> {
const packsToSearch: string[] = [];
// The CLI can handle both library packs and query packs, so search both packs in order.
packsToSearch.push(qlpacks.dbschemePack);
if (qlpacks.queryPack !== undefined) {
packsToSearch.push(qlpacks.queryPack);
}
const queries = await resolveQueriesFromPacks(cli, packsToSearch, keyType);
if (queries.length > 0) {
return queries;
}
// No queries found. Determine the correct error message for the various scenarios.
const keyTypeName = nameOfKeyType(keyType);
const keyTypeTag = tagOfKeyType(keyType);
const joinedPacksToSearch = packsToSearch.join(", ");
const error = redactableError`No ${keyTypeName} queries (tagged "${keyTypeTag}") could be found in the \
current library path (tried searching the following packs: ${joinedPacksToSearch}). \
Try upgrading the CodeQL libraries. If that doesn't work, then ${keyTypeName} queries are not yet available \
for this language.`;
void showAndLogExceptionWithTelemetry(extLogger, telemetryListener, error);
throw error;
}
async function resolveContextualQuery(
cli: CodeQLCliServer,
query: string,
): Promise<{ packPath: string; createdTempLockFile: boolean }> {
// Contextual queries now live within the standard library packs.
// This simplifies distribution (you don't need the standard query pack to use the AST viewer),
// but if the library pack doesn't have a lockfile, we won't be able to find
// other pack dependencies of the library pack.
// Work out the enclosing pack.
const packContents = await cli.packPacklist(query, false);
const packFilePath = packContents.find((p) =>
QLPACK_FILENAMES.includes(basename(p)),
);
if (packFilePath === undefined) {
// Should not happen; we already resolved this query.
throw new Error(
`Could not find a CodeQL pack file for the pack enclosing the contextual query ${query}`,
);
}
const packPath = dirname(packFilePath);
const lockFilePath = packContents.find((p) =>
["codeql-pack.lock.yml", "qlpack.lock.yml"].includes(basename(p)),
);
let createdTempLockFile = false;
if (!lockFilePath) {
// No lock file, likely because this library pack is in the package cache.
// Create a lock file so that we can resolve dependencies and library path
// for the contextual query.
void extLogger.log(
`Library pack ${packPath} is missing a lock file; creating a temporary lock file`,
);
await cli.packResolveDependencies(packPath);
createdTempLockFile = true;
// Clear CLI server pack cache before installing dependencies,
// so that it picks up the new lock file, not the previously cached pack.
void extLogger.log("Clearing the CodeQL CLI server's pack cache");
await cli.clearCache();
// Install dependencies.
void extLogger.log(
`Installing package dependencies for library pack ${packPath}`,
);
await cli.packInstall(packPath);
}
return { packPath, createdTempLockFile };
}
async function removeTemporaryLockFile(packPath: string) {
const tempLockFilePath = resolve(packPath, "codeql-pack.lock.yml");
void extLogger.log(
`Deleting temporary package lock file at ${tempLockFilePath}`,
);
// It's fine if the file doesn't exist.
await promises.rm(resolve(packPath, "codeql-pack.lock.yml"), {
force: true,
return resolveLocalQueries(cli, qlpacks, nameOfKeyType(keyType), {
kind: kindOfKeyType(keyType),
"tags contain": [tagOfKeyType(keyType)],
});
}
@@ -178,10 +37,7 @@ export async function runContextualQuery(
token: CancellationToken,
templates: Record<string, string>,
): Promise<CoreCompletedQuery> {
const { packPath, createdTempLockFile } = await resolveContextualQuery(
cli,
query,
);
const { cleanup } = await createLockFileForStandardQuery(cli, query);
const queryRun = qs.createQueryRun(
db.databaseUri.fsPath,
{ queryPath: query, quickEvalPosition: undefined },
@@ -200,8 +56,6 @@ export async function runContextualQuery(
token,
new TeeLogger(qs.logger, queryRun.outputDir.logPath),
);
if (createdTempLockFile) {
await removeTemporaryLockFile(packPath);
}
await cleanup?.();
return results;
}

7
extensions/ql-vscode/src/language-support/contextual/template-provider.ts
View File

@@ -27,11 +27,7 @@ import {
SELECTED_SOURCE_LINE,
SELECTED_SOURCE_COLUMN,
} from "./location-finder";
import {
qlpackOfDatabase,
resolveQueries,
runContextualQuery,
} from "./query-resolver";
import { resolveQueries, runContextualQuery } from "./query-resolver";
import {
isCanary,
NO_CACHE_AST_VIEWER,
@@ -39,6 +35,7 @@ import {
} from "../../config";
import { CoreCompletedQuery, QueryRunner } from "../../query-server";
import { AstBuilder } from "../ast-viewer/ast-builder";
import { qlpackOfDatabase } from "../../local-queries";
/**
* Runs templated CodeQL queries to find definitions in

1
extensions/ql-vscode/src/local-queries/index.ts
View File

@@ -1,5 +1,6 @@
export * from "./local-queries";
export * from "./local-query-run";
export * from "./query-resolver";
export * from "./quick-eval-code-lens-provider";
export * from "./quick-query";
export * from "./results-view";

131
extensions/ql-vscode/src/local-queries/query-resolver.ts Normal file
View File

@@ -0,0 +1,131 @@
import { CodeQLCliServer } from "../codeql-cli/cli";
import { DatabaseItem } from "../databases/local-databases";
import {
getPrimaryDbscheme,
getQlPackForDbscheme,
QlPacksForLanguage,
} from "../databases/qlpack";
import { file } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump } from "js-yaml";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { redactableError } from "../common/errors";
import { showAndLogExceptionWithTelemetry } from "../common/logging";
import { extLogger } from "../common/logging/vscode";
import { telemetryListener } from "../common/vscode/telemetry";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
db: Pick<DatabaseItem, "contents">,
): Promise<QlPacksForLanguage> {
if (db.contents === undefined) {
throw new Error("Database is invalid and cannot infer QLPack.");
}
const datasetPath = db.contents.datasetUri.fsPath;
const dbscheme = await getPrimaryDbscheme(datasetPath);
return await getQlPackForDbscheme(cli, dbscheme);
}
export interface QueryConstraints {
kind?: string;
"tags contain"?: string[];
"tags contain all"?: string[];
}
/**
* Finds the queries with the specified kind and tags in a list of CodeQL packs.
*
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param constraints Constraints on the queries to search for.
* @returns The found queries from the first pack in which any matching queries were found.
*/
async function resolveQueriesFromPacks(
cli: CodeQLCliServer,
qlpacks: string[],
constraints: QueryConstraints,
): Promise<string[]> {
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
for (const qlpack of qlpacks) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: constraints,
});
}
await writeFile(
suiteFile,
dump(suiteYaml, {
noRefs: true, // CodeQL doesn't really support refs
}),
"utf8",
);
return await cli.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
}
/**
* Finds the queries with the specified kind and tags in a QLPack.
*
* @param cli The CLI instance to use.
* @param qlpacks The list of packs to search.
* @param name The name of the query to use in error messages.
* @param constraints Constraints on the queries to search for.
* @returns The found queries from the first pack in which any matching queries were found.
*/
export async function resolveQueries(
cli: CodeQLCliServer,
qlpacks: QlPacksForLanguage,
name: string,
constraints: QueryConstraints,
): Promise<string[]> {
const packsToSearch: string[] = [];
// The CLI can handle both library packs and query packs, so search both packs in order.
packsToSearch.push(qlpacks.dbschemePack);
if (qlpacks.queryPack !== undefined) {
packsToSearch.push(qlpacks.queryPack);
}
const queries = await resolveQueriesFromPacks(
cli,
packsToSearch,
constraints,
);
if (queries.length > 0) {
return queries;
}
// No queries found. Determine the correct error message for the various scenarios.
const humanConstraints = [];
if (constraints.kind !== undefined) {
humanConstraints.push(`kind "${constraints.kind}"`);
}
if (constraints["tags contain"] !== undefined) {
humanConstraints.push(`tagged "${constraints["tags contain"].join(" ")}"`);
}
if (constraints["tags contain all"] !== undefined) {
humanConstraints.push(
`tagged all of "${constraints["tags contain all"].join(" ")}"`,
);
}
const joinedPacksToSearch = packsToSearch.join(", ");
const error = redactableError`No ${name} queries (${humanConstraints.join(
", ",
)}) could be found in the \
current library path (tried searching the following packs: ${joinedPacksToSearch}). \
Try upgrading the CodeQL libraries. If that doesn't work, then ${name} queries are not yet available \
for this language.`;
void showAndLogExceptionWithTelemetry(extLogger, telemetryListener, error);
throw error;
}

77
extensions/ql-vscode/src/local-queries/standard-queries.ts Normal file
View File

@@ -0,0 +1,77 @@
import { CodeQLCliServer } from "../codeql-cli/cli";
import { QLPACK_FILENAMES, QLPACK_LOCK_FILENAMES } from "../common/ql";
import { basename, dirname, resolve } from "path";
import { extLogger } from "../common/logging/vscode";
import { promises } from "fs-extra";
import { BaseLogger } from "../common/logging";
type LockFileForStandardQueryResult = {
cleanup?: () => Promise<void>;
};
/**
* Create a temporary query suite for a given query living within the standard library packs.
*
* This will create a lock file so the CLI can run the query without having the ql submodule.
*/
export async function createLockFileForStandardQuery(
cli: CodeQLCliServer,
queryPath: string,
logger: BaseLogger = extLogger,
): Promise<LockFileForStandardQueryResult> {
// These queries live within the standard library packs.
// This simplifies distribution (you don't need the standard query pack to use the AST viewer),
// but if the library pack doesn't have a lockfile, we won't be able to find
// other pack dependencies of the library pack.
// Work out the enclosing pack.
const packContents = await cli.packPacklist(queryPath, false);
const packFilePath = packContents.find((p) =>
QLPACK_FILENAMES.includes(basename(p)),
);
if (packFilePath === undefined) {
// Should not happen; we already resolved this query.
throw new Error(
`Could not find a CodeQL pack file for the pack enclosing the contextual query ${queryPath}`,
);
}
const packPath = dirname(packFilePath);
const lockFilePath = packContents.find((p) =>
QLPACK_LOCK_FILENAMES.includes(basename(p)),
);
let cleanup: (() => Promise<void>) | undefined = undefined;
if (!lockFilePath) {
// No lock file, likely because this library pack is in the package cache.
// Create a lock file so that we can resolve dependencies and library path
// for the contextual query.
void logger.log(
`Library pack ${packPath} is missing a lock file; creating a temporary lock file`,
);
await cli.packResolveDependencies(packPath);
cleanup = async () => {
const tempLockFilePath = resolve(packPath, "codeql-pack.lock.yml");
void logger.log(
`Deleting temporary package lock file at ${tempLockFilePath}`,
);
// It's fine if the file doesn't exist.
await promises.rm(resolve(packPath, "codeql-pack.lock.yml"), {
force: true,
});
};
// Clear CLI server pack cache before installing dependencies,
// so that it picks up the new lock file, not the previously cached pack.
void logger.log("Clearing the CodeQL CLI server's pack cache");
await cli.clearCache();
// Install dependencies.
void logger.log(
`Installing package dependencies for library pack ${packPath}`,
);
await cli.packInstall(packPath);
}
return { cleanup };
}

20
extensions/ql-vscode/src/stories/common/LastUpdated.stories.tsx
View File

@@ -1,20 +0,0 @@
import * as React from "react";
import { Meta, StoryFn } from "@storybook/react";
import { LastUpdated as LastUpdatedComponent } from "../../view/common/LastUpdated";
export default {
title: "Last Updated",
component: LastUpdatedComponent,
} as Meta<typeof LastUpdatedComponent>;
const Template: StoryFn<typeof LastUpdatedComponent> = (args) => (
<LastUpdatedComponent {...args} />
);
export const LastUpdated = Template.bind({});
LastUpdated.args = {
lastUpdated: new Date(Date.now() - 3_600_000).toISOString(), // 1 hour ago
};

12
extensions/ql-vscode/src/variant-analysis/shared/variant-analysis-filter-sort.ts
View File

@@ -1,5 +1,4 @@
import { Repository, RepositoryWithMetadata } from "./repository";
import { parseDate } from "../../common/date";
import { assertNever } from "../../common/helpers-pure";
export enum FilterKey {
@@ -10,7 +9,6 @@ export enum FilterKey {
export enum SortKey {
Alphabetically = "alphabetically",
Popularity = "popularity",
MostRecentCommit = "mostRecentCommit",
NumberOfResults = "numberOfResults",
}
@@ -81,16 +79,6 @@ export function compareRepository(
}
}
// Newest to oldest
if (filterSortState?.sortKey === SortKey.MostRecentCommit) {
const lastUpdated =
(parseDate(right.updatedAt)?.getTime() ?? 0) -
(parseDate(left.updatedAt)?.getTime() ?? 0);
if (lastUpdated !== 0) {
return lastUpdated;
}
}
// Fall back on name compare. Use en-US because the repository name does not contain
// special characters due to restrictions in GitHub owner/repository names.
return left.fullName.localeCompare(right.fullName, "en-US", {

42
extensions/ql-vscode/src/view/common/LastUpdated.tsx
View File

@@ -1,42 +0,0 @@
import * as React from "react";
import { useMemo } from "react";
import styled from "styled-components";
import { parseDate } from "../../common/date";
import { humanizeRelativeTime } from "../../common/time";
import { Codicon } from "./icon";
const IconContainer = styled.span`
flex-grow: 0;
text-align: right;
margin-right: 0;
`;
const Duration = styled.span`
display: inline-block;
text-align: left;
width: 8em;
margin-left: 0.5em;
`;
type Props = {
lastUpdated?: string | null;
};
export const LastUpdated = ({ lastUpdated }: Props) => {
const date = useMemo(() => parseDate(lastUpdated), [lastUpdated]);
if (!date) {
return null;
}
return (
<div>
<IconContainer>
<Codicon name="repo-push" label="Most recent commit" />
</IconContainer>
<Duration>{humanizeRelativeTime(date.getTime() - Date.now())}</Duration>
</div>
);
};

42
extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx
View File

@@ -1,9 +1,6 @@
import * as React from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import {
ShowProgressMessage,
ToDataExtensionsEditorMessage,
} from "../../common/interface-types";
import { ToDataExtensionsEditorMessage } from "../../common/interface-types";
import {
VSCodeButton,
VSCodeCheckbox,
@@ -72,17 +69,6 @@ const ButtonsContainer = styled.div`
margin-bottom: 1rem;
`;
type ProgressBarProps = {
completion: number;
};
const ProgressBar = styled.div<ProgressBarProps>`
height: 10px;
width: ${(props) => props.completion * 100}%;
background-color: var(--vscode-progressBar-background);
`;
type Props = {
initialViewState?: DataExtensionEditorViewState;
initialExternalApiUsages?: ExternalApiUsage[];
@@ -108,11 +94,6 @@ export function DataExtensionsEditor({
const [modeledMethods, setModeledMethods] = useState<
Record<string, ModeledMethod>
>(initialModeledMethods);
const [progress, setProgress] = useState<Omit<ShowProgressMessage, "t">>({
step: 0,
maxStep: 0,
message: "",
});
useEffect(() => {
const listener = (evt: MessageEvent) => {
@@ -125,9 +106,6 @@ export function DataExtensionsEditor({
case "setExternalApiUsages":
setExternalApiUsages(msg.externalApiUsages);
break;
case "showProgress":
setProgress(msg);
break;
case "loadModeledMethods":
setModeledMethods((oldModeledMethods) => {
return {
@@ -274,21 +252,12 @@ export function DataExtensionsEditor({
});
}, [viewState?.mode]);
if (viewState === undefined) {
if (viewState === undefined || externalApiUsages.length === 0) {
return <LoadingContainer>Loading...</LoadingContainer>;
}
return (
<DataExtensionsEditorContainer>
{progress.maxStep > 0 && (
<p>
<ProgressBar completion={progress.step / progress.maxStep} />{" "}
{progress.message}
</p>
)}
{externalApiUsages.length > 0 && (
<>
<HeaderContainer>
<HeaderColumn>
<HeaderRow>
@@ -313,10 +282,7 @@ export function DataExtensionsEditor({
</LinkIconButton>
{viewState.enableFrameworkMode && (
<LinkIconButton onClick={onSwitchModeClick}>
<span
slot="start"
className="codicon codicon-library"
></span>
<span slot="start" className="codicon codicon-library"></span>
{viewState.mode === Mode.Framework
? "Model as application"
: "Model as dependency"}
@@ -361,8 +327,6 @@ export function DataExtensionsEditor({
onModelDependencyClick={onModelDependencyClick}
/>
</EditorContainer>
</>
)}
</DataExtensionsEditorContainer>
);
}

2
extensions/ql-vscode/src/view/variant-analysis/RepoRow.tsx
View File

@@ -24,7 +24,6 @@ import {
import { vscode } from "../vscode-api";
import { AnalyzedRepoItemContent } from "./AnalyzedRepoItemContent";
import StarCount from "../common/StarCount";
import { LastUpdated } from "../common/LastUpdated";
import { useTelemetryOnChange } from "../common/telemetry";
import { DeterminateProgressRing } from "../common/DeterminateProgressRing";
@@ -297,7 +296,6 @@ export const RepoRow = ({
<div>
<StarCount starCount={repository.stargazersCount} />
</div>
<LastUpdated lastUpdated={repository.updatedAt} />
</MetadataContainer>
</TitleContainer>
{isExpanded && expandableContentLoaded && (

3
extensions/ql-vscode/src/view/variant-analysis/RepositoriesSort.tsx
View File

@@ -34,9 +34,6 @@ export const RepositoriesSort = ({ value, onChange, className }: Props) => {
Number of results
</VSCodeOption>
<VSCodeOption value={SortKey.Popularity}>Popularity</VSCodeOption>
<VSCodeOption value={SortKey.MostRecentCommit}>
Most recent commit
</VSCodeOption>
</Dropdown>
);
};

31
extensions/ql-vscode/src/view/variant-analysis/__tests__/RepoRow.spec.tsx
View File

@@ -38,10 +38,7 @@ describe(RepoRow.name, () => {
expect(
screen.queryByRole("img", {
// There should not be any icons, except for the icons which are always shown
name: (name) =>
!["expand", "stars count", "most recent commit"].includes(
name.toLowerCase(),
),
name: (name) => !["expand", "stars count"].includes(name.toLowerCase()),
}),
).not.toBeInTheDocument();
@@ -279,26 +276,7 @@ describe(RepoRow.name, () => {
).toBeInTheDocument();
});
it("shows updated at", () => {
render({
repository: {
...createMockRepositoryWithMetadata(),
// 1 month ago
updatedAt: new Date(
Date.now() - 1000 * 60 * 60 * 24 * 30,
).toISOString(),
},
});
expect(screen.getByText("last month")).toBeInTheDocument();
expect(
screen.getByRole("img", {
name: "Most recent commit",
}),
).toBeInTheDocument();
});
it("does not show star count and updated at when unknown", () => {
it("does not show star count when unknown", () => {
render({
repository: {
id: undefined,
@@ -312,11 +290,6 @@ describe(RepoRow.name, () => {
name: "Stars count",
}),
).not.toBeInTheDocument();
expect(
screen.queryByRole("img", {
name: "Most recent commit",
}),
).not.toBeInTheDocument();
});
it("can expand the repo item", async () => {

75
extensions/ql-vscode/test/unit-tests/variant-analysis-filter-sort.test.ts
View File

@@ -204,55 +204,6 @@ describe(compareRepository.name, () => {
).toBeLessThan(0);
});
});
describe("when sort key is 'Most recent commit'", () => {
const sorter = compareRepository({
...permissiveFilterSortState,
sortKey: SortKey.MostRecentCommit,
});
const left = {
fullName: "github/galaxy",
updatedAt: "2020-01-01T00:00:00Z",
};
const right = {
fullName: "github/world",
updatedAt: "2021-01-01T00:00:00Z",
};
it("compares correctly", () => {
expect(sorter(left, right)).toBeGreaterThan(0);
});
it("compares the inverse correctly", () => {
expect(sorter(right, left)).toBeLessThan(0);
});
it("compares equal values correctly", () => {
expect(sorter(left, left)).toBe(0);
});
it("compares equal single values correctly", () => {
expect(
sorter(left, {
...right,
updatedAt: left.updatedAt,
}),
).toBeLessThan(0);
});
it("compares missing single values correctly", () => {
expect(
sorter(
{
...left,
updatedAt: undefined,
},
right,
),
).toBeGreaterThan(0);
});
});
});
describe(compareWithResults.name, () => {
@@ -303,32 +254,6 @@ describe(compareWithResults.name, () => {
});
});
describe("when sort key is 'Most recent commit'", () => {
const sorter = compareWithResults({
...permissiveFilterSortState,
sortKey: SortKey.MostRecentCommit,
});
const left = {
repository: {
id: 11,
fullName: "github/galaxy",
updatedAt: "2020-01-01T00:00:00Z",
},
};
const right = {
repository: {
id: 12,
fullName: "github/world",
updatedAt: "2021-01-01T00:00:00Z",
},
};
it("compares correctly", () => {
expect(sorter(left, right)).toBeGreaterThan(0);
});
});
describe("when sort key is results count", () => {
const sorter = compareWithResults({
...permissiveFilterSortState,

52
extensions/ql-vscode/test/vscode-tests/no-workspace/language-support/contextual/query-resolver.test.ts
View File

@@ -5,23 +5,11 @@ import { getErrorMessage } from "../../../../../src/common/helpers-pure";
import * as log from "../../../../../src/common/logging/notifications";
import * as workspaceFolders from "../../../../../src/common/vscode/workspace-folders";
import * as qlpack from "../../../../../src/databases/qlpack";
import {
KeyType,
qlpackOfDatabase,
resolveQueries,
} from "../../../../../src/language-support";
import { KeyType, resolveQueries } from "../../../../../src/language-support";
import { CodeQLCliServer } from "../../../../../src/codeql-cli/cli";
import { mockDatabaseItem, mockedObject } from "../../../utils/mocking.helpers";
import { mockedObject } from "../../../utils/mocking.helpers";
describe("queryResolver", () => {
let getQlPackForDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getQlPackForDbscheme
>;
let getPrimaryDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getPrimaryDbscheme
>;
const resolveQueriesInSuite = jest.fn();
const mockCli = mockedObject<CodeQLCliServer>({
@@ -29,16 +17,6 @@ describe("queryResolver", () => {
});
beforeEach(() => {
getQlPackForDbschemeSpy = jest
.spyOn(qlpack, "getQlPackForDbscheme")
.mockResolvedValue({
dbschemePack: "dbschemePack",
dbschemePackIsLibraryPack: false,
});
getPrimaryDbschemeSpy = jest
.spyOn(qlpack, "getPrimaryDbscheme")
.mockResolvedValue("primaryDbscheme");
jest
.spyOn(workspaceFolders, "getOnDiskWorkspaceFolders")
.mockReturnValue([]);
@@ -68,7 +46,7 @@ describe("queryResolver", () => {
queries: ".",
include: {
kind: "definitions",
"tags contain": "ide-contextual-queries/local-definitions",
"tags contain": ["ide-contextual-queries/local-definitions"],
},
},
]);
@@ -87,31 +65,9 @@ describe("queryResolver", () => {
expect(true).toBe(false);
} catch (e) {
expect(getErrorMessage(e)).toBe(
'No definitions queries (tagged "ide-contextual-queries/local-definitions") could be found in the current library path (tried searching the following packs: my-qlpack). Try upgrading the CodeQL libraries. If that doesn\'t work, then definitions queries are not yet available for this language.',
'No definitions queries (kind "definitions", tagged "ide-contextual-queries/local-definitions") could be found in the current library path (tried searching the following packs: my-qlpack). Try upgrading the CodeQL libraries. If that doesn\'t work, then definitions queries are not yet available for this language.',
);
}
});
});
describe("qlpackOfDatabase", () => {
it("should get the qlpack of a database", async () => {
getQlPackForDbschemeSpy.mockResolvedValue({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
const db = mockDatabaseItem({
contents: {
datasetUri: {
fsPath: "/path/to/database",
},
},
});
const result = await qlpackOfDatabase(mockCli, db);
expect(result).toEqual({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
expect(getPrimaryDbschemeSpy).toBeCalledWith("/path/to/database");
});
});
});

119
extensions/ql-vscode/test/vscode-tests/no-workspace/local-queries/query-resolver.test.ts Normal file
View File

@@ -0,0 +1,119 @@
import {
qlpackOfDatabase,
resolveQueries,
} from "../../../../src/local-queries";
import { mockDatabaseItem, mockedObject } from "../../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import * as qlpack from "../../../../src/databases/qlpack";
import * as workspaceFolders from "../../../../src/common/vscode/workspace-folders";
import * as log from "../../../../src/common/logging/notifications";
import { load } from "js-yaml";
import * as fs from "fs-extra";
describe("qlpackOfDatabase", () => {
let getQlPackForDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getQlPackForDbscheme
>;
let getPrimaryDbschemeSpy: jest.SpiedFunction<
typeof qlpack.getPrimaryDbscheme
>;
const mockCli = mockedObject<CodeQLCliServer>({});
beforeEach(() => {
getQlPackForDbschemeSpy = jest
.spyOn(qlpack, "getQlPackForDbscheme")
.mockResolvedValue({
dbschemePack: "dbschemePack",
dbschemePackIsLibraryPack: false,
});
getPrimaryDbschemeSpy = jest
.spyOn(qlpack, "getPrimaryDbscheme")
.mockResolvedValue("primaryDbscheme");
});
it("should get the qlpack of a database", async () => {
getQlPackForDbschemeSpy.mockResolvedValue({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
const db = mockDatabaseItem({
contents: {
datasetUri: {
fsPath: "/path/to/database",
},
},
});
const result = await qlpackOfDatabase(mockCli, db);
expect(result).toEqual({
dbschemePack: "my-qlpack",
dbschemePackIsLibraryPack: false,
});
expect(getPrimaryDbschemeSpy).toBeCalledWith("/path/to/database");
});
});
describe("resolveQueries", () => {
const resolveQueriesInSuite = jest.fn();
const mockCli = mockedObject<CodeQLCliServer>({
resolveQueriesInSuite,
});
beforeEach(() => {
jest
.spyOn(workspaceFolders, "getOnDiskWorkspaceFolders")
.mockReturnValue([]);
jest.spyOn(log, "showAndLogErrorMessage").mockResolvedValue(undefined);
});
it("should resolve a query", async () => {
resolveQueriesInSuite.mockReturnValue(["a", "b"]);
const result = await resolveQueries(
mockCli,
{ dbschemePack: "my-qlpack", dbschemePackIsLibraryPack: false },
"my query",
{
kind: "graph",
"tags contain": ["ide-contextual-queries/print-ast"],
},
);
expect(result).toEqual(["a", "b"]);
expect(resolveQueriesInSuite).toHaveBeenCalledWith(
expect.stringMatching(/\.qls$/),
[],
);
const fileName = resolveQueriesInSuite.mock.calls[0][0];
expect(load(await fs.readFile(fileName, "utf-8"))).toEqual([
{
from: "my-qlpack",
queries: ".",
include: {
kind: "graph",
"tags contain": ["ide-contextual-queries/print-ast"],
},
},
]);
});
it("should throw an error when there are no queries found", async () => {
resolveQueriesInSuite.mockReturnValue([]);
await expect(
resolveQueries(
mockCli,
{ dbschemePack: "my-qlpack", dbschemePackIsLibraryPack: false },
"my query",
{
kind: "graph",
"tags contain": ["ide-contextual-queries/print-ast"],
},
),
).rejects.toThrowError(
'No my query queries (kind "graph", tagged "ide-contextual-queries/print-ast") could be found in the current library path (tried searching the following packs: my-qlpack). Try upgrading the CodeQL libraries. If that doesn\'t work, then my query queries are not yet available for this language.',
);
});
});

113
extensions/ql-vscode/test/vscode-tests/no-workspace/local-queries/standard-queries.test.ts Normal file
View File

@@ -0,0 +1,113 @@
import { mockedObject } from "../../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import { dir, DirectoryResult } from "tmp-promise";
import { join } from "path";
import { createLockFileForStandardQuery } from "../../../../src/local-queries/standard-queries";
import { outputFile, pathExists } from "fs-extra";
describe("createLockFileForStandardQuery", () => {
let tmpDir: DirectoryResult;
let packPath: string;
let qlpackPath: string;
let queryPath: string;
const packPacklist = jest.fn();
const packResolveDependencies = jest.fn();
const clearCache = jest.fn();
const packInstall = jest.fn();
const mockCli = mockedObject<CodeQLCliServer>({
packPacklist,
packResolveDependencies,
clearCache,
packInstall,
});
beforeEach(async () => {
tmpDir = await dir({
unsafeCleanup: true,
});
packPath = join(tmpDir.path, "a", "b");
qlpackPath = join(packPath, "qlpack.yml");
queryPath = join(packPath, "d", "e", "query.ql");
packPacklist.mockResolvedValue([qlpackPath, queryPath]);
});
afterEach(async () => {
await tmpDir.cleanup();
});
describe("when the lock file exists", () => {
let lockfilePath: string;
beforeEach(async () => {
lockfilePath = join(packPath, "qlpack.lock.yml");
packPacklist.mockResolvedValue([qlpackPath, lockfilePath, queryPath]);
});
it("does not resolve or install dependencies", async () => {
expect(await createLockFileForStandardQuery(mockCli, queryPath)).toEqual({
cleanup: undefined,
});
expect(packResolveDependencies).not.toHaveBeenCalled();
expect(clearCache).not.toHaveBeenCalled();
expect(packInstall).not.toHaveBeenCalled();
});
it("does not resolve or install dependencies with a codeql-pack.lock.yml", async () => {
lockfilePath = join(packPath, "codeql-pack.lock.yml");
packPacklist.mockResolvedValue([qlpackPath, lockfilePath, queryPath]);
expect(await createLockFileForStandardQuery(mockCli, queryPath)).toEqual({
cleanup: undefined,
});
expect(packResolveDependencies).not.toHaveBeenCalled();
expect(clearCache).not.toHaveBeenCalled();
expect(packInstall).not.toHaveBeenCalled();
});
});
describe("when the lock file does not exist", () => {
it("resolves and installs dependencies", async () => {
expect(await createLockFileForStandardQuery(mockCli, queryPath)).toEqual({
cleanup: expect.any(Function),
});
expect(packResolveDependencies).toHaveBeenCalledWith(packPath);
expect(clearCache).toHaveBeenCalledWith();
expect(packInstall).toHaveBeenCalledWith(packPath);
});
it("cleans up the lock file using the cleanup function", async () => {
const { cleanup } = await createLockFileForStandardQuery(
mockCli,
queryPath,
);
expect(cleanup).not.toBeUndefined();
const lockfilePath = join(packPath, "codeql-pack.lock.yml");
await outputFile(lockfilePath, "lock file contents");
await cleanup?.();
expect(await pathExists(lockfilePath)).toBe(false);
});
it("does not fail when cleaning up a non-existing lock file", async () => {
const { cleanup } = await createLockFileForStandardQuery(
mockCli,
queryPath,
);
expect(cleanup).not.toBeUndefined();
await cleanup?.();
});
});
});