From 742bca1cf5fbc915683adf0c24d824313c2904ad Mon Sep 17 00:00:00 2001 From: Andrew Eisenberg Date: Thu, 18 Nov 2021 15:27:29 -0800 Subject: [PATCH] Remote queries: Handle nested queries This change allows remote queries to run a query from a directory that is not in the root of the qlpack. The change is the following: 1. walk up the directory hierarchy to check for a non-local qlpack.yml 2. Copy over the files as before, but keep track of the relative location of the query compared to the location of the qlpack.yml. 3. Change the defaultSuite of the qlpack.yml so that _only_ this query is run as part of the default query. Also, this adds a new integration test to ensure the nested query is packaged appropriately. --- extensions/ql-vscode/src/run-remote-query.ts | 58 ++++++++-- .../data-remote-qlpack-nested/not-in-pack.ql | 2 + .../otherfolder/lib.qll | 3 + .../data-remote-qlpack-nested/qlpack.yml | 4 + .../subfolder/in-pack.ql | 4 + .../data-remote-qlpack/qlpack.yml | 1 - .../cli-integration/run-remote-query.test.ts | 102 +++++++++++++++--- 7 files changed, 150 insertions(+), 24 deletions(-) create mode 100644 extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/not-in-pack.ql create mode 100644 extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/otherfolder/lib.qll create mode 100644 extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/qlpack.yml create mode 100644 extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/subfolder/in-pack.ql diff --git a/extensions/ql-vscode/src/run-remote-query.ts b/extensions/ql-vscode/src/run-remote-query.ts index b3bba8308..6f19dc3bc 100644 --- a/extensions/ql-vscode/src/run-remote-query.ts +++ b/extensions/ql-vscode/src/run-remote-query.ts @@ -11,13 +11,19 @@ import { getRemoteControllerRepo, getRemoteRepositoryLists, setRemoteControllerR import { tmpDir } from './run-queries'; import { ProgressCallback, UserCancellationException } from './commandRunner'; import { OctokitResponse } from '@octokit/types/dist-types'; - interface Config { repositories: string[]; ref?: string; language?: string; } +interface QlPack { + name: string; + version: string; + dependencies: { [key: string]: string }; + defaultSuite?: Record; + defaultSuiteFile?: Record; +} interface RepoListQuickPickItem extends QuickPickItem { repoList: string[]; } @@ -89,13 +95,9 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri base64Pack: string, language: string }> { - const originalPackRoot = path.dirname(queryFile); - // TODO this assumes that the qlpack.yml is in the same directory as the query file, but in reality, - // the file could be in a parent directory. - const targetQueryFileName = path.join(queryPackDir, path.basename(queryFile)); - - // the server is expecting the query file to be named `query.ql`. Rename it here. - const renamedQueryFile = path.join(queryPackDir, 'query.ql'); + const originalPackRoot = await findPackRoot(queryFile); + const packRelativePath = path.relative(originalPackRoot, queryFile); + const targetQueryFileName = path.join(queryPackDir, packRelativePath); let language: string | undefined; if (await fs.pathExists(path.join(originalPackRoot, 'qlpack.yml'))) { @@ -138,8 +140,6 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri // copy only the query file to the query pack directory // and generate a synthetic query pack - // TODO this has a limitation that query packs inside of a workspace will not resolve its peer dependencies. - // Something to work on later. For now, we will only support query packs that are not in a workspace. void logger.log(`Copying ${queryFile} to ${queryPackDir}`); await fs.copy(queryFile, targetQueryFileName); void logger.log('Generating synthetic query pack'); @@ -156,7 +156,8 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri throw new UserCancellationException('Could not determine language.'); } - await fs.rename(targetQueryFileName, renamedQueryFile); + // fix the default suite of the query pack dir + await fixDefaultSuite(queryPackDir, packRelativePath); const bundlePath = await getPackedBundlePath(queryPackDir); void logger.log(`Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`); @@ -189,6 +190,21 @@ async function ensureQueryPackName(queryPackDir: string) { } } +async function findPackRoot(queryFile: string): Promise { + // recursively find the directory containing qlpack.yml + let dir = path.dirname(queryFile); + while (!(await fs.pathExists(path.join(dir, 'qlpack.yml')))) { + dir = path.dirname(dir); + if (dir === '/') { + // there is no qlpack.yml in this direcory or any parent directory. + // just use the query file's directory as the pack root. + return path.dirname(queryFile); + } + } + + return dir; +} + async function createRemoteQueriesTempDirectory() { const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true }); const queryPackDir = path.join(remoteQueryDir.path, 'query-pack'); @@ -413,3 +429,23 @@ export async function attemptRerun( void showAndLogErrorMessage(error); } } + +/** + * Updates the default suite of the query pack. This is used to ensure + * only the specified query is run. + * + * @param queryPackDir The directory containing the query pack + * @param packRelativePath The relative path to the query pack from the root of the query pack + */ +async function fixDefaultSuite(queryPackDir: string, packRelativePath: string): Promise { + const packPath = path.join(queryPackDir, 'qlpack.yml'); + const qlpack = (await yaml.safeLoad(await fs.readFile(packPath, 'utf8'))) as QlPack; + delete qlpack.defaultSuite; + delete qlpack.defaultSuiteFile; + + qlpack.defaultSuite = { + description: 'Query suite for remote query', + query: packRelativePath + }; + await fs.writeFile(packPath, yaml.safeDump(qlpack)); +} diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/not-in-pack.ql b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/not-in-pack.ql new file mode 100644 index 000000000..7ca7d397b --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/not-in-pack.ql @@ -0,0 +1,2 @@ +// This file should not be included the remote query pack. +select 1 diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/otherfolder/lib.qll b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/otherfolder/lib.qll new file mode 100644 index 000000000..08a900299 --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/otherfolder/lib.qll @@ -0,0 +1,3 @@ +int number() { + result = 1 +} diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/qlpack.yml b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/qlpack.yml new file mode 100644 index 000000000..d43fadff5 --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/qlpack.yml @@ -0,0 +1,4 @@ +name: github/remote-query-pack +version: 0.0.0 +dependencies: + codeql/javascript-all: '*' diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/subfolder/in-pack.ql b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/subfolder/in-pack.ql new file mode 100644 index 000000000..a3d6982f2 --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/subfolder/in-pack.ql @@ -0,0 +1,4 @@ +import javascript +import otherfolder.lib + +select number() diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml index d715c36c3..d43fadff5 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml @@ -1,5 +1,4 @@ name: github/remote-query-pack version: 0.0.0 -extractor: javascript dependencies: codeql/javascript-all: '*' diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts index 2afee4800..fcc1bcea6 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts @@ -13,7 +13,7 @@ import { CodeQLExtensionInterface } from '../../extension'; import { setRemoteControllerRepo, setRemoteRepositoryLists } from '../../config'; import { UserCancellationException } from '../../commandRunner'; -describe('Remote queries', function() { +describe.only('Remote queries', function() { const baseDir = path.join(__dirname, '../../../src/vscode-tests/cli-integration'); let sandbox: sinon.SinonSandbox; @@ -80,8 +80,7 @@ describe('Remote queries', function() { const queryPackDir = path.join(queryPackRootDir, 'query-pack'); printDirectoryContents(queryPackDir); - // in-pack.ql renamed to query.ql - expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true; + expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true; expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true; expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true; @@ -96,7 +95,7 @@ describe('Remote queries', function() { const compiledPackDir = path.join(queryPackDir, '.codeql/pack/github/remote-query-pack/0.0.0/'); printDirectoryContents(compiledPackDir); - expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true; + expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true; expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true; expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true; // depending on the cli version, we should have one of these files @@ -105,6 +104,7 @@ describe('Remote queries', function() { fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml')) ).to.be.true; expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false; + verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', 'github/remote-query-pack', '0.0.0'); // dependencies const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql'); @@ -129,8 +129,8 @@ describe('Remote queries', function() { const queryPackDir = path.join(queryPackRootDir, 'query-pack'); printDirectoryContents(queryPackDir); - // in-pack.ql renamed to query.ql - expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true; + + expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true; expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true; // depending on the cli version, we should have one of these files expect( @@ -143,8 +143,10 @@ describe('Remote queries', function() { // the compiled pack const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/'); printDirectoryContents(compiledPackDir); - expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true; + expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true; expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true; + verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', 'codeql-remote/query', '0.0.0'); + // depending on the cli version, we should have one of these files expect( fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) || @@ -165,6 +167,60 @@ describe('Remote queries', function() { expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']); }); + it('should run a remote query that is nested inside a qlpack', async () => { + const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql'); + + const queryPackRootDir = (await runRemoteQuery(cli, credentials, fileUri, true, progress, token))!; + + // to retrieve the list of repositories + expect(showQuickPickSpy).to.have.been.calledOnce; + + // check a few files that we know should exist and others that we know should not + + // the tarball to deliver to the server + printDirectoryContents(queryPackRootDir); + expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined; + + const queryPackDir = path.join(queryPackRootDir, 'query-pack'); + printDirectoryContents(queryPackDir); + + expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true; + expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true; + // depending on the cli version, we should have one of these files + expect( + fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) || + fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml')) + ).to.be.true; + expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true; + expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false; + + // the compiled pack + const compiledPackDir = path.join(queryPackDir, '.codeql/pack/github/remote-query-pack/0.0.0/'); + printDirectoryContents(compiledPackDir); + expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true; + expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true; + expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true; + verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', 'github/remote-query-pack', '0.0.0'); + + // depending on the cli version, we should have one of these files + expect( + fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) || + fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml')) + ).to.be.true; + expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false; + // should have generated a correct qlpack file + const qlpackContents: any = yaml.safeLoad(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8')); + expect(qlpackContents.name).to.equal('github/remote-query-pack'); + expect(qlpackContents.version).to.equal('0.0.0'); + expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*'); + + // dependencies + const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql'); + printDirectoryContents(libraryDir); + const packNames = fs.readdirSync(libraryDir).sort(); + expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']); + }); + it('should cancel a run before uploading', async () => { const fileUri = getFile('data-remote-no-qlpack/in-pack.ql'); @@ -180,15 +236,37 @@ describe('Remote queries', function() { } }); + function verifyQlPack(qlpackPath: string, queryPath: string, packName: string, packVersion: string) { + const qlPack = yaml.safeLoad(fs.readFileSync(qlpackPath, 'utf8')); + + // don't check the build metadata since it is variable + delete (qlPack as any).buildMetadata; + + expect(qlPack).to.deep.equal({ + name: packName, + version: packVersion, + dependencies: { + 'codeql/javascript-all': '*', + }, + library: false, + defaultSuite: [{ + description: 'Query suite for remote query', + query: queryPath, + }] + }); + } + function getFile(file: string): Uri { return Uri.file(path.join(baseDir, file)); } function printDirectoryContents(dir: string) { - console.log(`DIR ${dir}`); - if (!fs.existsSync(dir)) { - console.log(`DIR ${dir} does not exist`); - } - fs.readdirSync(dir).sort().forEach(f => console.log(` ${f}`)); + dir; + // uncomment to debug + // console.log(`DIR ${dir}`); + // if (!fs.existsSync(dir)) { + // console.log(`DIR ${dir} does not exist`); + // } + // fs.readdirSync(dir).sort().forEach(f => console.log(` ${f}`)); } });