mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #1340 from github/aeisenberg/fix-integration-tests

Browse Source 
Fix cli-integration tests
This commit is contained in:
Andrew Eisenberg
2022-05-09 14:48:17 -07:00
committed by GitHub
parent 12d6ea3966 1dc663339d
commit c058e7a128
25 changed files with 798 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.vscode/launch.json vendored
View File

@@ -12,7 +12,6 @@
// Add a reference to a workspace to open. Eg-
// "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
],
"stopOnEntry": false,
"sourceMaps": true,
"outFiles": [
"${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -46,7 +45,6 @@
"ts-node/register",
"test/pure-tests/**/*.ts"
],
"port": 9229,
"stopOnEntry": false,
"sourceMaps": true,
"console": "integratedTerminal",
@@ -60,10 +58,10 @@
"args": [
"--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
"--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
"--disable-workspace-trust",
"--disable-extensions",
"--disable-gpu"
],
"stopOnEntry": false,
"sourceMaps": true,
"outFiles": [
"${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -77,11 +75,11 @@
"args": [
"--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
"--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
"--disable-workspace-trust",
"--disable-extensions",
"--disable-gpu",
"${workspaceRoot}/extensions/ql-vscode/test/data"
],
"stopOnEntry": false,
"sourceMaps": true,
"outFiles": [
"${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -95,6 +93,7 @@
"args": [
"--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
"--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
"--disable-workspace-trust",
"--disable-gpu",
"--disable-extension",
"eamodio.gitlens",
@@ -121,7 +120,6 @@
// This option overrides the CLI_VERSION option.
// "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
},
"stopOnEntry": false,
"sourceMaps": true,
"outFiles": [
"${workspaceRoot}/extensions/ql-vscode/out/**/*.js",

790
extensions/ql-vscode/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
extensions/ql-vscode/package.json
View File

@@ -1229,6 +1229,6 @@
]
},
"resolutions": {
"glob-parent": "~6.0.0"
"glob-parent": "6.0.0"
}
}

4
extensions/ql-vscode/src/config.ts
View File

@@ -368,3 +368,7 @@ const ACTION_BRANCH = new Setting('actionBranch', REMOTE_QUERIES_SETTING);
export function getActionBranch(): string {
return ACTION_BRANCH.getValue<string>() || 'main';
}
export function isIntegrationTestMode() {
return process.env.INTEGRATION_TEST_MODE === 'true';
}

8
extensions/ql-vscode/src/telemetry.ts
View File

@@ -1,6 +1,6 @@
import { ConfigurationTarget, Extension, ExtensionContext, ConfigurationChangeEvent } from 'vscode';
import TelemetryReporter from 'vscode-extension-telemetry';
import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY } from './config';
import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY, isIntegrationTestMode } from './config';
import * as appInsights from 'applicationinsights';
import { logger } from './logging';
import { UserCancellationException } from './commandRunner';
@@ -162,7 +162,11 @@ export class TelemetryListener extends ConfigListener {
if (!this.wasTelemetryRequested()) {
// if global telemetry is disabled, avoid showing the dialog or making any changes
let result = undefined;
if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
if (
GLOBAL_ENABLE_TELEMETRY.getValue() &&
// Avoid showing the dialog if we are in integration test mode.
!isIntegrationTestMode()
) {
// Extension won't start until this completes.
result = await showBinaryChoiceWithUrlDialog(
'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?',

4
extensions/ql-vscode/src/vscode-tests/cli-integration/data/codeql-pack.lock.yml Normal file
View File

@@ -0,0 +1,4 @@
---
dependencies: {}
compiled: false
lockVersion: 1.0.0

19
extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts
View File

@@ -9,7 +9,7 @@ import { CodeQLCliServer } from '../../cli';
import { DatabaseManager } from '../../databases';
import { promptImportLgtmDatabase, importArchiveDatabase, promptImportInternetDatabase } from '../../databaseFetcher';
import { ProgressCallback } from '../../commandRunner';
import { dbLoc, DB_URL, storagePath } from './global.helper';
import { cleanDatabases, dbLoc, DB_URL, storagePath } from './global.helper';
/**
* Run various integration tests for databases
@@ -27,6 +27,14 @@ describe('Databases', function() {
beforeEach(async () => {
try {
sandbox = sinon.createSandbox();
// the uri.fsPath function on windows returns a lowercase drive letter
// so, force the storage path string to be lowercase, too.
progressCallback = sandbox.spy();
inputBoxStub = sandbox.stub(window, 'showInputBox');
sandbox.stub(window, 'showErrorMessage');
sandbox.stub(window, 'showInformationMessage');
const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
if ('databaseManager' in extension) {
databaseManager = extension.databaseManager;
@@ -34,19 +42,16 @@ describe('Databases', function() {
throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
}
sandbox = sinon.createSandbox();
// the uri.fsPath function on windows returns a lowercase drive letter
// so, force the storage path string to be lowercase, too.
progressCallback = sandbox.spy();
inputBoxStub = sandbox.stub(window, 'showInputBox');
await cleanDatabases(databaseManager);
} catch (e) {
fail(e as Error);
}
});
afterEach(() => {
afterEach(async () => {
try {
sandbox.restore();
await cleanDatabases(databaseManager);
} catch (e) {
fail(e as Error);
}

14
extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts
View File

@@ -4,13 +4,19 @@ import * as fs from 'fs-extra';
import fetch from 'node-fetch';
import { fail } from 'assert';
import { ConfigurationTarget, extensions, workspace } from 'vscode';
import { commands, ConfigurationTarget, extensions, workspace } from 'vscode';
import { CodeQLExtensionInterface } from '../../extension';
import { DatabaseManager } from '../../databases';
// This file contains helpers shared between actual tests.
export const DB_URL = 'https://github.com/github/vscode-codeql/files/5586722/simple-db.zip';
process.addListener('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise ', p, ' reason: ', reason);
fail(String(reason));
});
// We need to resolve the path, but the final three segments won't exist until later, so we only resolve the
// first portion of the path.
export const dbLoc = path.join(fs.realpathSync(path.join(__dirname, '../../../')), 'build/tests/db.zip');
@@ -84,3 +90,9 @@ export default function(mocha: Mocha) {
}
);
}
export async function cleanDatabases(databaseManager: DatabaseManager) {
for (const item of databaseManager.databaseItems) {
await commands.executeCommand('codeQLDatabases.removeDatabase', item);
}
}

2
extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts
View File

@@ -1,4 +1,5 @@
import 'source-map-support/register';
import 'vscode-test';
import { runTestsInDirectory } from '../index-template';
import 'mocha';
import * as sinonChai from 'sinon-chai';
@@ -8,7 +9,6 @@ import * as chaiAsPromised from 'chai-as-promised';
chai.use(chaiAsPromised);
chai.use(sinonChai);
// The simple database used throughout the tests
export function run(): Promise<void> {
return runTestsInDirectory(__dirname, true);
}

22
extensions/ql-vscode/src/vscode-tests/cli-integration/packaging.test.ts
View File

@@ -27,6 +27,17 @@ describe('Packaging commands', function() {
beforeEach(async function() {
sandbox = sinon.createSandbox();
progress = sandbox.spy();
quickPickSpy = sandbox.stub(window, 'showQuickPick');
inputBoxSpy = sandbox.stub(window, 'showInputBox');
showAndLogErrorMessageSpy = sandbox.stub();
showAndLogInformationMessageSpy = sandbox.stub();
mod = proxyquire('../../packaging', {
'./helpers': {
showAndLogErrorMessage: showAndLogErrorMessageSpy,
showAndLogInformationMessage: showAndLogInformationMessageSpy,
},
});
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -45,17 +56,6 @@ describe('Packaging commands', function() {
}. Skipping this test.`);
this.skip();
}
progress = sandbox.spy();
quickPickSpy = sandbox.stub(window, 'showQuickPick');
inputBoxSpy = sandbox.stub(window, 'showInputBox');
showAndLogErrorMessageSpy = sandbox.stub();
showAndLogInformationMessageSpy = sandbox.stub();
mod = proxyquire('../../packaging', {
'./helpers': {
showAndLogErrorMessage: showAndLogErrorMessageSpy,
showAndLogInformationMessage: showAndLogInformationMessageSpy,
},
});
});
afterEach(() => {

21
extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts
View File

@@ -8,7 +8,7 @@ import * as yaml from 'js-yaml';
import { DatabaseItem, DatabaseManager } from '../../databases';
import { CodeQLExtensionInterface } from '../../extension';
import { dbLoc, storagePath } from './global.helper';
import { cleanDatabases, dbLoc, storagePath } from './global.helper';
import { importArchiveDatabase } from '../../databaseFetcher';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../../run-queries';
import { CodeQLCliServer } from '../../cli';
@@ -52,16 +52,21 @@ describe('Queries', function() {
qs = extension.qs;
cli.quiet = true;
ctx = extension.ctx;
qlpackFile = `${ctx.storagePath}/quick-queries/qlpack.yml`;
qlFile = `${ctx.storagePath}/quick-queries/quick-query.ql`;
qlpackFile = `${ctx.storageUri?.fsPath}/quick-queries/qlpack.yml`;
qlFile = `${ctx.storageUri?.fsPath}/quick-queries/quick-query.ql`;
} else {
throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
}
// Ensure we are starting from a clean slate.
safeDel(qlFile);
safeDel(qlpackFile);
progress = sandbox.spy();
token = {} as CancellationToken;
// Add a database
// Add a database, but make sure the database manager is empty first
await cleanDatabases(databaseManager);
const uri = Uri.file(dbLoc);
const maybeDbItem = await importArchiveDatabase(
uri.toString(true),
@@ -81,9 +86,12 @@ describe('Queries', function() {
}
});
afterEach(() => {
afterEach(async () => {
try {
sandbox.restore();
safeDel(qlpackFile);
safeDel(qlFile);
await cleanDatabases(databaseManager);
} catch (e) {
fail(e as Error);
}
@@ -135,9 +143,6 @@ describe('Queries', function() {
});
it('should create a quick query', async () => {
safeDel(qlFile);
safeDel(qlpackFile);
await commands.executeCommand('codeQL.quickQuery');
// should have created the quick query file and query pack file

41
extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts
View File

@@ -9,6 +9,7 @@ import { skipIfNoCodeQL } from '../ensureCli';
import { getOnDiskWorkspaceFolders, getQlPackForDbscheme, languageToDbScheme } from '../../helpers';
import { resolveQueries } from '../../contextual/queryResolver';
import { KeyType } from '../../contextual/keyType';
import { fail } from 'assert';
/**
* Perform proper integration tests by running the CLI
@@ -74,24 +75,28 @@ describe('Use cli', function() {
it('should resolve printAST queries for supported languages', async function() {
skipIfNoCodeQL(this);
supportedLanguages.forEach(async lang => {
if (lang === 'go') {
// The codeql-go submodule is not available in the integration tests.
return;
try {
for (const lang of supportedLanguages) {
if (lang === 'go') {
// The codeql-go submodule is not available in the integration tests.
return;
}
console.log(`resolving printAST queries for ${lang}`);
const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]);
expect(pack.dbschemePack).to.contain(lang);
if (pack.dbschemePackIsLibraryPack) {
expect(pack.queryPack).to.contain(lang);
}
const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
// It doesn't matter what the name or path of the query is, only
// that we have found exactly one query.
expect(result.length).to.eq(1);
}
console.log(`resolving printAST queries for ${lang}`);
const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]);
expect(pack.dbschemePack).to.contain(lang);
if (pack.dbschemePackIsLibraryPack) {
expect(pack.queryPack).to.contain(lang);
}
const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
// It doesn't matter what the name or path of the query is, only
// that we have found exactly one query.
expect(result.length).to.eq(1);
});
} catch (e) {
fail(e as Error);
}
});
});

1
extensions/ql-vscode/src/vscode-tests/minimal-workspace/config.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as Sinon from 'sinon';
import { expect } from 'chai';
import { workspace } from 'vscode';

1
extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as sinon from 'sinon';
import * as tmp from 'tmp';
import * as fs from 'fs-extra';

6
extensions/ql-vscode/src/vscode-tests/minimal-workspace/index.ts
View File

@@ -1,10 +1,12 @@
import 'source-map-support/register';
import { runTestsInDirectory } from '../index-template';
import 'vscode-test';
import * as sinonChai from 'sinon-chai';
import * as chai from 'chai';
import 'chai/register-should';
import * as chaiAsPromised from 'chai-as-promised';
import { runTestsInDirectory } from '../index-template';
chai.use(chaiAsPromised);
chai.use(sinonChai);

1
extensions/ql-vscode/src/vscode-tests/minimal-workspace/qltest-discovery.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import { Uri, WorkspaceFolder } from 'vscode';
import { expect } from 'chai';
import * as fs from 'fs-extra';

1
extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/fileRangeFromURI.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import { expect } from 'chai';
import { Uri, Range } from 'vscode';

1
extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as yaml from 'js-yaml';
import * as sinon from 'sinon';
import { expect } from 'chai';

1
extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as sinon from 'sinon';
import * as path from 'path';
import * as fs from 'fs-extra';

1
extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as tmp from 'tmp';
import * as path from 'path';
import * as fs from 'fs-extra';

4
extensions/ql-vscode/src/vscode-tests/no-workspace/index.ts
View File

@@ -1,10 +1,12 @@
import 'source-map-support/register';
import { runTestsInDirectory } from '../index-template';
import 'vscode-test';
import * as sinonChai from 'sinon-chai';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import 'chai/register-should';
import { runTestsInDirectory } from '../index-template';
chai.use(chaiAsPromised);
chai.use(sinonChai);

1
extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/repository-selection.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as sinon from 'sinon';
import { expect } from 'chai';
import { window } from 'vscode';

1
extensions/ql-vscode/src/vscode-tests/no-workspace/test-adapter.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import * as sinon from 'sinon';
import * as fs from 'fs-extra';
import { Uri, WorkspaceFolder } from 'vscode';

22
extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
View File

@@ -7,16 +7,13 @@ import {
resolveCliPathFromVSCodeExecutablePath
} from 'vscode-test';
import { assertNever } from '../pure/helpers-pure';
import * as tmp from 'tmp-promise';
// For some reason, `TestOptions` is not exported directly from `vscode-test`,
// For some reason, the following are not exported directly from `vscode-test`,
// but we can be tricky and import directly from the out file.
import { TestOptions } from 'vscode-test/out/runTest';
// Which version of vscode to test against. Can set to 'stable' or
// 'insiders' or an explicit version number. See runTest.d.ts in
// vscode-test for more details.
// For CI purposes we want to leave this at 'stable' to catch any bugs
// that might show up with new vscode versions released, even though
// this makes testing not-quite-pure, but it can be changed for local
@@ -63,6 +60,8 @@ async function runTestsWithRetryOnSegfault(suite: TestOptions, tries: number): P
process.exit(1);
}
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
/**
* Integration test runner. Launches the VSCode Extension Development Host with this extension installed.
* See https://github.com/microsoft/vscode-test/blob/master/sample/test/runTest.ts
@@ -76,7 +75,7 @@ async function main() {
// Which tests to run. Use a comma-separated list of directories.
const testDirsString = process.argv[2];
const dirs = testDirsString.split(',').map(dir => dir.trim().toLocaleLowerCase());
const extensionTestsEnv: Record<string, string> = {};
if (dirs.includes(TestDir.CliIntegration)) {
console.log('Installing required extensions');
const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath);
@@ -93,6 +92,7 @@ async function main() {
stdio: 'inherit',
}
);
extensionTestsEnv.INTEGRATION_TEST_MODE = 'true';
}
console.log(`Running integration tests in these directories: ${dirs}`);
@@ -105,11 +105,13 @@ async function main() {
vscodeExecutablePath,
extensionDevelopmentPath,
extensionTestsPath: path.resolve(__dirname, dir, 'index'),
extensionTestsEnv,
launchArgs
}, 3);
}
} catch (err) {
console.error(`Unexpected exception while running tests: ${err}`);
console.error((err as Error).stack);
process.exit(1);
}
}
@@ -122,13 +124,15 @@ function getLaunchArgs(dir: TestDir) {
case TestDir.NoWorksspace:
return [
'--disable-extensions',
'--disable-gpu'
'--disable-gpu',
'--user-data-dir=' + path.join(tmpDir.name, dir, 'user-data')
];
case TestDir.MinimalWorksspace:
return [
'--disable-extensions',
'--disable-gpu',
'--user-data-dir=' + path.join(tmpDir.name, dir, 'user-data'),
path.resolve(__dirname, '../../test/data')
];
@@ -145,8 +149,8 @@ function getLaunchArgs(dir: TestDir) {
'github.codespaces',
'--disable-extension',
'github.copilot',
process.env.TEST_CODEQL_PATH!
];
'--user-data-dir=' + path.join(tmpDir.name, dir, 'user-data'),
].concat(process.env.TEST_CODEQL_PATH ? [process.env.TEST_CODEQL_PATH] : []);
default:
assertNever(dir);

1
extensions/ql-vscode/test/pure-tests/sarif-processing.test.ts
View File

@@ -1,4 +1,3 @@
import 'vscode-test';
import 'mocha';
import * as chaiAsPromised from 'chai-as-promised';
import * as chai from 'chai';