mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3087 from github/koesie10/drop-support-cli

Browse Source 
Drop support for CLI versions < v2.11.6
This commit is contained in:
Koen Vlaswinkel
2023-11-24 10:32:12 +01:00
committed by GitHub
parent 424e8d3145 ec192affdc
commit c3425b5146
11 changed files with 22 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/CHANGELOG.md
View File

@@ -3,6 +3,7 @@
## [UNRELEASED]
- Add a prompt to the "Quick query" command to encourage users in single-folder workspaces to use "Create query" instead. [#3082](https://github.com/github/vscode-codeql/pull/3082)
- Remove support for CodeQL CLI versions older than 2.11.6. [#3087](https://github.com/github/vscode-codeql/pull/3087)
## 1.10.0 - 16 November 2023

88
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -881,10 +881,9 @@ export class CodeQLCliServer implements Disposable {
additionalPacks: string[],
queryPath: string,
): Promise<MlModelsInfo> {
const args = (await this.cliConstraints.supportsPreciseResolveMlModels())
? // use the dirname of the path so that we can handle query libraries
[...this.getAdditionalPacksArg(additionalPacks), dirname(queryPath)]
: this.getAdditionalPacksArg(additionalPacks);
const args =
// use the dirname of the path so that we can handle query libraries
[...this.getAdditionalPacksArg(additionalPacks), dirname(queryPath)];
return await this.runJsonCodeQlCliCommand<MlModelsInfo>(
["resolve", "ml-models"],
args,
@@ -988,9 +987,7 @@ export class CodeQLCliServer implements Disposable {
const subcommandArgs = [
"--format=text",
`--end-summary=${endSummaryPath}`,
...((await this.cliConstraints.supportsSourceMap())
? ["--sourcemap"]
: []),
"--sourcemap",
inputPath,
outputPath,
];
@@ -1712,41 +1709,7 @@ export function shouldDebugCliServer() {
export class CliVersionConstraint {
// The oldest version of the CLI that we support. This is used to determine
// whether to show a warning about the CLI being too old on startup.
public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.9.4");
/**
* CLI version where building QLX packs for remote queries is supported.
* (The options were _accepted_ by a few earlier versions, but only from
* 2.11.3 will it actually use the existing compilation cache correctly).
*/
public static CLI_VERSION_QLX_REMOTE = new SemVer("2.11.3");
/**
* CLI version where the `resolve ml-models` subcommand was enhanced to work with packaging.
*/
public static CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS = new SemVer(
"2.10.0",
);
/**
* CLI version where the `resolve extensions` subcommand exists.
*/
public static CLI_VERSION_WITH_RESOLVE_EXTENSIONS = new SemVer("2.10.2");
/**
* CLI version that supports the `--sourcemap` option for log generation.
*/
public static CLI_VERSION_WITH_SOURCEMAP = new SemVer("2.10.3");
/**
* CLI version that supports the new query server.
*/
public static CLI_VERSION_WITH_NEW_QUERY_SERVER = new SemVer("2.11.1");
/**
* CLI version that supports `${workspace}` references in qlpack.yml files.
*/
public static CLI_VERSION_WITH_WORKSPACE_RFERENCES = new SemVer("2.11.3");
public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.11.6");
/**
* CLI version that supports the `--kind` option for the `resolve qlpacks` command.
@@ -1796,48 +1759,9 @@ export class CliVersionConstraint {
return (await this.cli.getVersion()).compare(v) >= 0;
}
async supportsQlxRemote() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_QLX_REMOTE);
}
async supportsPreciseResolveMlModels() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS,
);
}
async supportsResolveExtensions() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS,
);
}
async supportsSourceMap() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_SOURCEMAP,
);
}
async supportsNewQueryServer() {
// This allows users to explicitly opt-out of the new query server.
return (
allowCanaryQueryServer() &&
this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER,
)
);
}
async supportsNewQueryServerForTests() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER,
);
}
async supportsWorkspaceReferences() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_WORKSPACE_RFERENCES,
);
return allowCanaryQueryServer();
}
async supportsQlpacksKind() {

10
extensions/ql-vscode/src/model-editor/model-editor-module.ts
View File

@@ -155,16 +155,6 @@ export class ModelEditorModule extends DisposableObject {
return;
}
if (
!(await this.cliServer.cliConstraints.supportsResolveExtensions())
) {
void showAndLogErrorMessage(
this.app.logger,
`This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS.format()} or later.`,
);
return;
}
const modelFile = await pickExtensionPack(
this.cliServer,
db,

6
extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
View File

@@ -105,7 +105,6 @@ async function generateQueryPack(
await cliServer.clearCache();
let precompilationOpts: string[] = [];
if (await cliServer.cliConstraints.supportsQlxRemote()) {
if (await cliServer.cliConstraints.usesGlobalCompilationCache()) {
precompilationOpts = ["--qlx"];
} else {
@@ -116,9 +115,6 @@ async function generateQueryPack(
`--compilation-cache=${ccache}`,
];
}
} else {
precompilationOpts = ["--no-precompile"];
}
if (await cliServer.useExtensionPacks()) {
await injectExtensionPacks(cliServer, queryPackDir, workspaceFolders);
@@ -540,7 +536,7 @@ async function getControllerRepoFromApi(
}
}
export function removeWorkspaceRefs(qlpack: QlPackFile) {
function removeWorkspaceRefs(qlpack: QlPackFile) {
if (!qlpack.dependencies) {
return;
}

2
extensions/ql-vscode/supported_cli_versions.json
View File

@@ -4,7 +4,5 @@
"v2.13.5",
"v2.12.7",
"v2.11.6",
"v2.9.4",
"v2.10.5",
"nightly"
]

4
extensions/ql-vscode/test/vscode-tests/cli-integration/debugger/debugger.test.ts
View File

@@ -179,10 +179,6 @@ describeWithCodeQL()("Debugger", () => {
});
it("should pass additionalArgs through to query server", async () => {
if (!(await cli.cliConstraints.supportsNewQueryServerForTests())) {
// Only works with the new query server.
return;
}
await withDebugController(appCommands, async (controller) => {
await controller.startDebugging(
{

275
extensions/ql-vscode/test/vscode-tests/cli-integration/legacy-query.test.ts
View File

@@ -1,275 +0,0 @@
import { existsSync } from "fs-extra";
import { join, basename } from "path";
import { dirSync } from "tmp";
import { pathToFileURL } from "url";
import { CancellationTokenSource } from "vscode-jsonrpc";
import * as messages from "../../../src/query-server/legacy-messages";
import * as qsClient from "../../../src/query-server/legacy/query-server-client";
import * as cli from "../../../src/codeql-cli/cli";
import { CellValue } from "../../../src/common/bqrs-cli-types";
import { describeWithCodeQL } from "../cli";
import { QueryServerClient } from "../../../src/query-server/legacy/query-server-client";
import {
extLogger,
ProgressReporter,
} from "../../../src/common/logging/vscode";
import { createMockApp } from "../../__mocks__/appMock";
import { getActivatedExtension } from "../global.helper";
const baseDir = join(__dirname, "../../../test/data");
const tmpDir = dirSync({
prefix: "query_test_",
keep: false,
unsafeCleanup: true,
});
const COMPILED_QUERY_PATH = join(tmpDir.name, "compiled.qlo");
const RESULTS_PATH = join(tmpDir.name, "results.bqrs");
const source = new CancellationTokenSource();
const token = source.token;
class Checkpoint<T> {
private res: () => void;
private rej: (e: Error) => void;
private promise: Promise<T>;
constructor() {
this.res = () => {
/**/
};
this.rej = () => {
/**/
};
this.promise = new Promise((res, rej) => {
this.res = res as () => Record<string, never>;
this.rej = rej;
});
}
async done(): Promise<T> {
return this.promise;
}
async resolve(): Promise<void> {
this.res();
}
async reject(e: Error): Promise<void> {
this.rej(e);
}
}
type ResultSets = {
[name: string]: CellValue[][];
};
type QueryTestCase = {
queryPath: string;
expectedResultSets: ResultSets;
};
// Test cases: queries to run and their expected results.
const queryTestCases: QueryTestCase[] = [
{
queryPath: join(baseDir, "query.ql"),
expectedResultSets: {
"#select": [[42, 3.14159, "hello world", true]],
},
},
{
queryPath: join(baseDir, "compute-default-strings.ql"),
expectedResultSets: {
"#select": [[{ label: "(no string representation)" }]],
},
},
{
queryPath: join(baseDir, "multiple-result-sets.ql"),
expectedResultSets: {
edges: [
[1, 2],
[2, 3],
],
"#select": [["s"]],
},
},
];
const db: messages.Dataset = {
dbDir: join(__dirname, "../../../.vscode-test/test-db"),
workingSet: "default",
};
describeWithCodeQL()("using the legacy query server", () => {
const nullProgressReporter: ProgressReporter = {
report: () => {
/** ignore */
},
};
let qs: qsClient.QueryServerClient;
let cliServer: cli.CodeQLCliServer;
let supportNewQueryServer = false;
beforeAll(async () => {
const extension = await getActivatedExtension();
cliServer = extension.cliServer;
cliServer.quiet = true;
if (await cliServer.cliConstraints.supportsNewQueryServerForTests()) {
console.log(
"Skipping legacy-query tests: the CLI supports the new query server",
);
supportNewQueryServer = true;
}
qs = new QueryServerClient(
createMockApp({}),
{
codeQlPath:
(await extension.distributionManager.getCodeQlPathWithoutVersionCheck()) ||
"",
debug: false,
cacheSize: 0,
numThreads: 1,
saveCache: false,
timeoutSecs: 0,
},
cliServer,
{
contextStoragePath: tmpDir.name,
logger: extLogger,
},
(task) => task(nullProgressReporter, new CancellationTokenSource().token),
);
await qs.startQueryServer();
});
for (const queryTestCase of queryTestCases) {
const queryName = basename(queryTestCase.queryPath);
const compilationSucceeded = new Checkpoint<void>();
const evaluationSucceeded = new Checkpoint<void>();
const parsedResults = new Checkpoint<void>();
it("should register the database if necessary", async () => {
if (supportNewQueryServer) {
return;
}
await qs.sendRequest(messages.registerDatabases, { databases: [db] });
});
it(`should be able to compile query ${queryName}`, async () => {
if (supportNewQueryServer) {
return;
}
expect(existsSync(queryTestCase.queryPath)).toBe(true);
try {
const qlProgram: messages.QlProgram = {
libraryPath: [],
dbschemePath: join(baseDir, "test.dbscheme"),
queryPath: queryTestCase.queryPath,
};
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true,
},
queryToCheck: qlProgram,
resultPath: COMPILED_QUERY_PATH,
target: { query: {} },
};
const result = await qs.sendRequest(
messages.compileQuery,
params,
token,
() => {
/**/
},
);
expect(result.messages!.length).toBe(0);
await compilationSucceeded.resolve();
} catch (e) {
await compilationSucceeded.reject(e as Error);
}
});
it(`should be able to run query ${queryName}`, async () => {
if (supportNewQueryServer) {
return;
}
try {
await compilationSucceeded.done();
const callbackId = qs.registerCallback((_res) => {
void evaluationSucceeded.resolve();
});
const queryToRun: messages.QueryToRun = {
resultsPath: RESULTS_PATH,
qlo: pathToFileURL(COMPILED_QUERY_PATH).toString(),
allowUnknownTemplates: true,
id: callbackId,
timeoutSecs: 1000,
};
const params: messages.EvaluateQueriesParams = {
db,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: true,
useSequenceHint: false,
};
await qs.sendRequest(messages.runQueries, params, token, () => {
/**/
});
} catch (e) {
await evaluationSucceeded.reject(e as Error);
}
});
const actualResultSets: ResultSets = {};
it(`should be able to parse results of query ${queryName}`, async () => {
if (supportNewQueryServer) {
return;
}
await evaluationSucceeded.done();
const info = await cliServer.bqrsInfo(RESULTS_PATH);
for (const resultSet of info["result-sets"]) {
const decoded = await cliServer.bqrsDecode(
RESULTS_PATH,
resultSet.name,
);
actualResultSets[resultSet.name] = decoded.tuples;
}
await parsedResults.resolve();
});
it(`should have correct results for query ${queryName}`, async () => {
if (supportNewQueryServer) {
return;
}
await parsedResults.done();
expect(actualResultSets).not.toEqual({});
expect(Object.keys(actualResultSets!).sort()).toEqual(
Object.keys(queryTestCase.expectedResultSets).sort(),
);
for (const name in queryTestCase.expectedResultSets) {
expect(actualResultSets![name]).toEqual(
queryTestCase.expectedResultSets[name],
);
}
});
}
});

16
extensions/ql-vscode/test/vscode-tests/cli-integration/model-editor/modeled-method-fs.test.ts
View File

@@ -135,10 +135,6 @@ describe("modeled-method-fs", () => {
describe("listModelFiles", () => {
it("should return the empty set when the extension pack is empty", async () => {
if (!(await cli.cliConstraints.supportsResolveExtensions())) {
return;
}
const extensionPackPath = writeExtensionPackFiles("extension-pack", []);
const modelFiles = await listModelFiles(extensionPackPath, cli);
@@ -146,10 +142,6 @@ describe("modeled-method-fs", () => {
});
it("should find all model files", async () => {
if (!(await cli.cliConstraints.supportsResolveExtensions())) {
return;
}
const extensionPackPath = writeExtensionPackFiles("extension-pack", [
"library1.model.yml",
"library2.model.yml",
@@ -165,10 +157,6 @@ describe("modeled-method-fs", () => {
});
it("should ignore model files from other extension packs", async () => {
if (!(await cli.cliConstraints.supportsResolveExtensions())) {
return;
}
const extensionPackPath = writeExtensionPackFiles("extension-pack", [
"library1.model.yml",
]);
@@ -183,10 +171,6 @@ describe("modeled-method-fs", () => {
describe("loadModeledMethods", () => {
it("should load modeled methods", async () => {
if (!(await cli.cliConstraints.supportsResolveExtensions())) {
return;
}
const extensionPackPath = writeExtensionPackFiles("extension-pack", [
"library.model.yml",
]);

24
extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts
View File

@@ -105,20 +105,12 @@ describeWithCodeQL()("using the new query server", () => {
let cliServer: cli.CodeQLCliServer;
let db: string;
let supportNewQueryServer = true;
beforeAll(async () => {
const app = createMockApp({});
const extension = await getActivatedExtension();
cliServer = extension.cliServer;
cliServer.quiet = true;
if (!(await cliServer.cliConstraints.supportsNewQueryServerForTests())) {
console.log(
"Skipping new-query tests: the CLI supports only the legacy query server",
);
supportNewQueryServer = false;
}
qs = new QueryServerClient(
app,
{
@@ -154,18 +146,10 @@ describeWithCodeQL()("using the new query server", () => {
const parsedResults = new Checkpoint<void>();
it("should register the database", async () => {
if (!supportNewQueryServer) {
return;
}
await qs.sendRequest(messages.registerDatabases, { databases: [db] });
});
it(`should be able to run query ${queryName}`, async () => {
if (!supportNewQueryServer) {
return;
}
try {
const params: messages.RunQueryParams = {
db,
@@ -193,10 +177,6 @@ describeWithCodeQL()("using the new query server", () => {
const actualResultSets: ResultSets = {};
it(`should be able to parse results of query ${queryName}`, async () => {
if (!supportNewQueryServer) {
return;
}
await evaluationSucceeded.done();
const info = await cliServer.bqrsInfo(RESULTS_PATH);
@@ -211,10 +191,6 @@ describeWithCodeQL()("using the new query server", () => {
});
it(`should have correct results for query ${queryName}`, async () => {
if (!supportNewQueryServer) {
return;
}
await parsedResults.done();
expect(actualResultSets).not.toEqual({});
expect(Object.keys(actualResultSets!).sort()).toEqual(

26
extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
View File

@@ -9,12 +9,7 @@ import {
CliVersionConstraint,
CodeQLCliServer,
} from "../../../../src/codeql-cli/cli";
import {
fixWorkspaceReferences,
getActivatedExtension,
restoreWorkspaceReferences,
storagePath,
} from "../../global.helper";
import { getActivatedExtension, storagePath } from "../../global.helper";
import { VariantAnalysisResultsManager } from "../../../../src/variant-analysis/variant-analysis-results-manager";
import {
VariantAnalysisStatus,
@@ -70,13 +65,9 @@ describe("Variant Analysis Manager", () => {
typeof ghApiClient.submitVariantAnalysis
>;
let mockApiResponse: VariantAnalysisApiResponse;
let originalDeps: Record<string, string> | undefined;
let executeCommandSpy: jest.SpiedFunction<typeof commands.executeCommand>;
const baseDir = join(__dirname, "..");
const qlpackFileWithWorkspaceRefs = getFile(
"data-remote-qlpack/qlpack.yml",
).fsPath;
beforeEach(async () => {
jest.spyOn(window, "showQuickPick").mockResolvedValueOnce(
@@ -107,19 +98,6 @@ describe("Variant Analysis Manager", () => {
// always run in the vscode-codeql repo
await setRemoteControllerRepo("github/vscode-codeql");
// Only new version support `${workspace}` in qlpack.yml
originalDeps = await fixWorkspaceReferences(
qlpackFileWithWorkspaceRefs,
cli,
);
});
afterEach(async () => {
await restoreWorkspaceReferences(
qlpackFileWithWorkspaceRefs,
originalDeps,
);
});
it("should run a variant analysis that is part of a qlpack", async () => {
@@ -360,11 +338,9 @@ describe("Variant Analysis Manager", () => {
expect(packFS.fileExists(file)).toBe(true);
});
if (await cli.cliConstraints.supportsQlxRemote()) {
qlxFilesThatExist.forEach((file) => {
expect(packFS.fileExists(file)).toBe(true);
});
}
filesThatDoNotExist.forEach((file) => {
expect(packFS.fileExists(file)).toBe(false);
});

56
extensions/ql-vscode/test/vscode-tests/global.helper.ts
View File

@@ -1,17 +1,14 @@
import { join } from "path";
import { load, dump } from "js-yaml";
import { realpathSync, readFileSync, writeFileSync } from "fs-extra";
import { Uri, extensions } from "vscode";
import { realpathSync } from "fs-extra";
import { extensions, Uri } from "vscode";
import {
DatabaseItem,
DatabaseManager,
} from "../../src/databases/local-databases";
import { CodeQLCliServer } from "../../src/codeql-cli/cli";
import { removeWorkspaceRefs } from "../../src/variant-analysis/run-remote-query";
import { CodeQLExtensionInterface } from "../../src/extension";
import { importArchiveDatabase } from "../../src/databases/database-fetcher";
import { createMockCommandManager } from "../__mocks__/commandsMock";
import { QlPackFile } from "../../src/packaging/qlpack-file";
// This file contains helpers shared between tests that work with an activated extension.
@@ -73,52 +70,3 @@ export async function getActivatedExtension(): Promise<CodeQLExtensionInterface>
export async function cleanDatabases(databaseManager: DatabaseManager) {
await databaseManager.removeAllDatabases();
}
/**
* Conditionally removes `${workspace}` references from a qlpack.yml or codeql-pack.yml file.
* CLI versions earlier than 2.11.3 do not support `${workspace}` references in the dependencies block.
* If workspace references are removed, the qlpack.yml or codeql-pack.yml file is re-written to disk
* without the `${workspace}` references and the original dependencies are returned.
*
* @param qlpackFileWithWorkspaceRefs The qlpack.yml or codeql-pack.yml file with workspace refs
* @param cli The cli to use to check version constraints
* @returns The original dependencies with workspace refs, or undefined if the CLI version supports workspace refs and no changes were made
*/
export async function fixWorkspaceReferences(
qlpackFileWithWorkspaceRefs: string,
cli: CodeQLCliServer,
): Promise<Record<string, string> | undefined> {
if (!(await cli.cliConstraints.supportsWorkspaceReferences())) {
// remove the workspace references from the qlpack
const qlpack = load(
readFileSync(qlpackFileWithWorkspaceRefs, "utf8"),
) as QlPackFile;
const originalDeps = { ...qlpack.dependencies };
removeWorkspaceRefs(qlpack);
writeFileSync(qlpackFileWithWorkspaceRefs, dump(qlpack));
return originalDeps;
}
return undefined;
}
/**
* Restores the original dependencies with `${workspace}` refs to a qlpack.yml or codeql-pack.yml file.
* See `fixWorkspaceReferences` for more details.
*
* @param qlpackFileWithWorkspaceRefs The qlpack.yml or codeql-pack.yml file to restore workspace refs
* @param originalDeps the original dependencies with workspace refs or undefined if
* no changes were made.
*/
export async function restoreWorkspaceReferences(
qlpackFileWithWorkspaceRefs: string,
originalDeps?: Record<string, string>,
) {
if (!originalDeps) {
return;
}
const qlpack = load(
readFileSync(qlpackFileWithWorkspaceRefs, "utf8"),
) as QlPackFile;
qlpack.dependencies = originalDeps;
writeFileSync(qlpackFileWithWorkspaceRefs, dump(qlpack));
}