import { existsSync, createWriteStream, mkdirpSync } from "fs-extra"; import { normalize, join } from "path"; import { getRequiredAssetName, extractZipArchive, codeQlLauncherName, } from "../../src/pure/distribution"; import fetch from "node-fetch"; import supportedCliVersions from "../../supported_cli_versions.json"; /** * This module ensures that the proper CLI is available for tests of the extension. * There are three environment variables to control this module: * * - CLI_VERSION: The version of the CLI to install. Defaults to the most recent * version. Note that for now, we must maintain the default version by hand. * This may be set to `nightly`, in which case the `NIGHTLY_URL` variable must * also be set. * * - NIGHTLY_URL: The URL for a nightly release of the CodeQL CLI that will be * used if `CLI_VERSION` is set to `nightly`. * * - CLI_BASE_DIR: The base dir where the CLI will be downloaded and unzipped. * The download location is `${CLI_BASE_DIR}/assets` and the unzip loction is * `${CLI_BASE_DIR}/${CLI_VERSION}` * * After downloading and unzipping, a new environment variable is set: * * - CLI_PATH: Points to the cli executable for the specified CLI_VERSION. This * is variable is available in the unit tests and will be used as the value * for `codeQL.cli.executablePath`. * * As an optimization, the cli will not be unzipped again if the executable already * exists. And the cli will not be re-downloaded if the zip already exists. */ const _1MB = 1024 * 1024; const _10MB = _1MB * 10; // CLI version to test. Use the latest supported version by default. // And be sure to update the env if it is not otherwise set. const CLI_VERSION = process.env.CLI_VERSION || supportedCliVersions[0]; process.env.CLI_VERSION = CLI_VERSION; // Base dir where CLIs will be downloaded into // By default, put it in the `build` directory in the root of the extension. const CLI_BASE_DIR = process.env.CLI_DIR || normalize(join(__dirname, "../../build/cli")); export async function ensureCli(useCli: boolean) { try { if (!useCli) { console.log("Not downloading CLI. It is not being used."); return; } if ("CLI_PATH" in process.env) { const executablePath = process.env.CLI_PATH; console.log(`Using existing CLI at ${executablePath}`); // The CLI_VERSION env variable is not used when the CLI_PATH is set. delete process.env.CLI_VERSION; return; } const assetName = getRequiredAssetName(); const url = getCliDownloadUrl(assetName); const unzipDir = getCliUnzipDir(); const downloadedFilePath = getDownloadFilePath(assetName); const executablePath = join( getCliUnzipDir(), "codeql", codeQlLauncherName(), ); // Use this environment variable to se to the `codeQL.cli.executablePath` in tests process.env.CLI_PATH = executablePath; if (existsSync(executablePath)) { console.log( `CLI version ${CLI_VERSION} is found ${executablePath}. Not going to download again.`, ); return; } if (!existsSync(downloadedFilePath)) { console.log( `CLI version ${CLI_VERSION} zip file not found. Downloading from '${url}' into '${downloadedFilePath}'.`, ); const assetStream = await fetch(url); const contentLength = Number( assetStream.headers.get("content-length") || 0, ); console.log("Total content size", Math.round(contentLength / _1MB), "MB"); const archiveFile = createWriteStream(downloadedFilePath); const body = assetStream.body; await new Promise((resolve, reject) => { let numBytesDownloaded = 0; let lastMessage = 0; body.on("data", (data) => { numBytesDownloaded += data.length; if (numBytesDownloaded - lastMessage > _10MB) { console.log( "Downloaded", Math.round(numBytesDownloaded / _1MB), "MB", ); lastMessage = numBytesDownloaded; } archiveFile.write(data); }); body.on("finish", () => { archiveFile.end(() => { console.log("Finished download into", downloadedFilePath); resolve(); }); }); body.on("error", reject); }); } else { console.log( `CLI version ${CLI_VERSION} zip file found at '${downloadedFilePath}'.`, ); } console.log(`Unzipping into '${unzipDir}'`); mkdirpSync(unzipDir); await extractZipArchive(downloadedFilePath, unzipDir); console.log("Done."); } catch (e) { console.error("Failed to download CLI."); console.error(e); process.exit(-1); } } /** * Url to download from */ function getCliDownloadUrl(assetName: string) { if (CLI_VERSION === "nightly") { if (!process.env.NIGHTLY_URL) throw new Error( "Nightly CLI was specified but no URL to download it from was given!", ); return `${process.env.NIGHTLY_URL}/${assetName}`; } return `https://github.com/github/codeql-cli-binaries/releases/download/${CLI_VERSION}/${assetName}`; } /** * Directory to place the downloaded cli into */ function getDownloadFilePath(assetName: string) { const dir = join(CLI_BASE_DIR, "assets", CLI_VERSION); mkdirpSync(dir); return join(dir, assetName); } /** * Directory to unzip the downloaded cli into. */ function getCliUnzipDir() { return join(CLI_BASE_DIR, CLI_VERSION); }